{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Copy of MA_GAN_estimation.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "machine_shape": "hm",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/BenYavor/MA_GAN/blob/master/Copy_of_MA_GAN_estimation.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2-49-RQG7bEV",
        "colab_type": "code",
        "outputId": "fe64b599-4816-4eeb-99fc-5dbbbfce64c6",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 412
        }
      },
      "source": [
        "!pip install tensorflow==2.0.0rc2\n",
        "#!pip install -q pyyaml h5py\n",
        "#!pip install -q tf_nightly\n",
        "import numpy as np\n",
        "%matplotlib inline\n",
        "import matplotlib.pyplot as plt   \n",
        "import warnings\n",
        "with warnings.catch_warnings():\n",
        "    warnings.filterwarnings(\"ignore\",category=FutureWarning)\n",
        "    import tensorflow as tf\n",
        "import os\n",
        "os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"1\" \n",
        "tf.__version__\n",
        "from tensorflow import keras\n",
        "import time\n",
        "from __future__ import absolute_import, division, print_function, unicode_literals"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: tensorflow==2.0.0rc2 in /usr/local/lib/python3.6/dist-packages (2.0.0rc2)\n",
            "Requirement already satisfied: gast==0.2.2 in /usr/local/lib/python3.6/dist-packages (from tensorflow==2.0.0rc2) (0.2.2)\n",
            "Requirement already satisfied: google-pasta>=0.1.6 in /usr/local/lib/python3.6/dist-packages (from tensorflow==2.0.0rc2) (0.1.7)\n",
            "Requirement already satisfied: keras-applications>=1.0.8 in /usr/local/lib/python3.6/dist-packages (from tensorflow==2.0.0rc2) (1.0.8)\n",
            "Requirement already satisfied: protobuf>=3.6.1 in /usr/local/lib/python3.6/dist-packages (from tensorflow==2.0.0rc2) (3.7.1)\n",
            "Requirement already satisfied: absl-py>=0.7.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow==2.0.0rc2) (0.8.0)\n",
            "Requirement already satisfied: tf-estimator-nightly<1.14.0.dev2019080602,>=1.14.0.dev2019080601 in /usr/local/lib/python3.6/dist-packages (from tensorflow==2.0.0rc2) (1.14.0.dev2019080601)\n",
            "Requirement already satisfied: termcolor>=1.1.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow==2.0.0rc2) (1.1.0)\n",
            "Requirement already satisfied: wrapt>=1.11.1 in /usr/local/lib/python3.6/dist-packages (from tensorflow==2.0.0rc2) (1.11.2)\n",
            "Requirement already satisfied: six>=1.10.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow==2.0.0rc2) (1.12.0)\n",
            "Requirement already satisfied: wheel>=0.26 in /usr/local/lib/python3.6/dist-packages (from tensorflow==2.0.0rc2) (0.33.6)\n",
            "Requirement already satisfied: opt-einsum>=2.3.2 in /usr/local/lib/python3.6/dist-packages (from tensorflow==2.0.0rc2) (3.1.0)\n",
            "Requirement already satisfied: keras-preprocessing>=1.0.5 in /usr/local/lib/python3.6/dist-packages (from tensorflow==2.0.0rc2) (1.1.0)\n",
            "Requirement already satisfied: tensorboard<2.1.0,>=2.0.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow==2.0.0rc2) (2.0.0)\n",
            "Requirement already satisfied: grpcio>=1.8.6 in /usr/local/lib/python3.6/dist-packages (from tensorflow==2.0.0rc2) (1.15.0)\n",
            "Requirement already satisfied: numpy<2.0,>=1.16.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow==2.0.0rc2) (1.16.5)\n",
            "Requirement already satisfied: astor>=0.6.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow==2.0.0rc2) (0.8.0)\n",
            "Requirement already satisfied: h5py in /usr/local/lib/python3.6/dist-packages (from keras-applications>=1.0.8->tensorflow==2.0.0rc2) (2.8.0)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.6/dist-packages (from protobuf>=3.6.1->tensorflow==2.0.0rc2) (41.2.0)\n",
            "Requirement already satisfied: werkzeug>=0.11.15 in /usr/local/lib/python3.6/dist-packages (from tensorboard<2.1.0,>=2.0.0->tensorflow==2.0.0rc2) (0.16.0)\n",
            "Requirement already satisfied: markdown>=2.6.8 in /usr/local/lib/python3.6/dist-packages (from tensorboard<2.1.0,>=2.0.0->tensorflow==2.0.0rc2) (3.1.1)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qa9PJHQS0UCJ",
        "colab_type": "text"
      },
      "source": [
        "## System funktionsweise Allgemeine Daten\n",
        "\n",
        "#### Rauschen\n",
        "genarats-> **shape**: batch_size * number_of_real_channels_uses_per_message \\\\\n",
        "and does a average power normalization\n",
        "\n",
        "\n",
        "#### Generator\n",
        "Eingang: (2*n,32)  ; Ausgang: (32,n)   \\\\\n",
        "Loss-Function:\n",
        "\n",
        "#### Discriminator\n",
        "Eingang: (2*n,32)  ; Ausgang: (32,1)  \\\\\n",
        "Loss-Function:\n",
        "\n",
        "\n",
        "#### Training\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4qpY-gawAf-9",
        "colab_type": "text"
      },
      "source": [
        "###Systemparameter\n",
        "$k$ - die Anzhal der bits \\\\\n",
        "$M$ - Anzahl der unterschiedlichen Nachrichten \\\\\n",
        "$n$ - channel uses **What is meant by that??** \\\\\n",
        "$N$ - Länge des Rauschvektors"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "86Y2r6qBAgKW",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "k = 4       # Number of information bits per message, i.e., M=2**k\n",
        "M = 2**k\n",
        "n = 4       # Number of real channel uses per message\n",
        "seed = 2    # Seed RNG reproduce identical results\n",
        "D_nb_weights = 32\n",
        "G_nb_weights = 32\n",
        "\n",
        "\n",
        "batch_size = 1000\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "x = tf.random.normal((batch_size,n))    #randomly sample input data (\"fake\" AE messages)\n",
        "x = x/tf.sqrt(2*tf.reduce_mean(tf.square(x))) #Average power normalization (not required if standard normal distribution is used )\n",
        "def make_zero(x):\n",
        "  return tf.keras.backend.zeros(shape=x.shape)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "cY9sHsfWT8By",
        "colab_type": "text"
      },
      "source": [
        "## Generator Model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uV7pjryDv4M4",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def EbNo2Sigma(ebnodb):\n",
        "    '''Convert Eb/No in dB to noise standard deviation'''\n",
        "    ebno = 10**(ebnodb/10)\n",
        "    return 1/np.sqrt(2*(2*k/n)*ebno)\n",
        "\n",
        "#numpy version of kl divergence\n",
        "def kl_divergence_np(p, q):\n",
        "    #use \"Laplace correction\" w to avoid zero and inf\n",
        "    w=1e-5\n",
        "    p = p + w\n",
        "    q = q + w\n",
        "    return np.sum(p * np.log(p / q))\n",
        "\n",
        "#tensorflow version of kl divergence\n",
        "def kl_divergence_tf(p, q):\n",
        "    #use \"Laplace correction\" w to avoid zero and inf\n",
        "    w = 1e-5\n",
        "    p = p + w\n",
        "    q = q + w\n",
        "    return tf.reduce_sum(p * tf.log(p / q))\n",
        "  \n",
        "  \n",
        "  \n",
        "train_SNR_dB = 7\n",
        "noise_std = EbNo2Sigma(train_SNR_dB)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hFMMLrY0LthL",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "randN_initial = keras.initializers.RandomNormal(mean=0.0, stddev=0.05, seed=None)\n",
        "#zero_initial = tf.keras.initializers.Zeros()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LXbS5lM9Tb9B",
        "colab_type": "code",
        "outputId": "30c88ebb-cfcf-485a-c95c-c702567d7d51",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 696
        }
      },
      "source": [
        "#\n",
        "\n",
        "#def generator(x):\n",
        "    # Concatenate z and y\n",
        "#    G_n = tf.random.normal([tf.shape(x)[0],n],dtype=tf.float32)  #create noise directly within the generator  \n",
        "#    inputs = tf.concat(values=[x, G_n], axis=1)\n",
        "    #dense NN\n",
        "#    G_h1 = tf.nn.relu(tf.matmul(inputs, G_W1) + G_b1)\n",
        "#    G_h2 = tf.nn.relu(tf.matmul(G_h1, G_W2) + G_b2)\n",
        "#    G_lin = tf.matmul(G_h2, G_W3) + G_b3\n",
        "    #G_prob = tf.nn.sigmoid(G_lin)\n",
        "#    return G_lin\n",
        "\n",
        "def generator_noise(x):\n",
        "  G_n = tf.random.normal([tf.shape(x)[0],n])  #create noise directly within the generator  \n",
        "  return G_n\n",
        "    \n",
        "tf.print(generator_noise(x).shape)\n",
        "\n",
        "#def get_generator(input = tf.keras.Input(shape=(batch_size,n)),training = False):\n",
        "#  model = tf.keras.Sequential()\n",
        "#  model.add(tf.keras.layers.Lambda(generator_noise))\n",
        "#  model.add(tf.keras.layers.Dense(32,use_bias=True, activation='relu'))#,input_shape=((2*n,))))\n",
        "#  model.add(tf.keras.layers.Dense(32,use_bias=True, activation='relu'))\n",
        "#  model.add(tf.keras.layers.Dense(n,use_bias=False, activation='sigmoid'))\n",
        "#  return model\n",
        "\n",
        "\n",
        "#input1 = tf.keras.layers.Input(shape=(n,))\n",
        "#x1 = tf.keras.layers.Dense(n, activation='relu')(input1)\n",
        "#input2 = tf.keras.layers.Input(shape=(n,)) #tf.keras.backend.zeros(shape=(n,))# Input(shape=(n,))\n",
        "#input3 = tf.keras.layers.GaussianNoise(noise_std)(input2) #input1,stddev =0.5\n",
        "#x2 = tf.keras.layers.Dense(n, activation='relu')(input3)\n",
        "#Equivalent to subtracted = keras.layers.subtract([x1, x2])\n",
        "#subtracted = tf.keras.layers.Concatenate(-1)([x1, x2])\n",
        "#h1 = tf.keras.layers.Dense(32)(subtracted)\n",
        "#h2 = tf.keras.layers.Dense(32,use_bias=True, activation='relu')(h1)\n",
        "#out = tf.keras.layers.Dense(n, use_bias= False, activation='relu')(h2)\n",
        "\n",
        "#tf.reshape(input,(tf.shape(input)[0],-1))\n",
        "input1 = tf.keras.layers.Input(shape=(n,))\n",
        "x1 = tf.keras.layers.Dense(n)(input1)\n",
        "input2 =tf.random.normal([tf.shape(input1)[0],n]) \n",
        "x2 = tf.keras.layers.Dense(n)(input2)\n",
        "subtracted = tf.keras.layers.Concatenate(1)([x1, x2])\n",
        "h1 = tf.keras.layers.Dense(32,use_bias=True,  activation='relu')(subtracted)\n",
        "h2 = tf.keras.layers.Dense(32,use_bias=True, activation='relu')(h1)\n",
        "out = tf.keras.layers.Dense(n, use_bias= True, activation='linear')(h2)\n",
        "\n",
        "#input2 = tf.keras.layers.Input(shape=(n,)) #tf.keras.backend.zeros(shape=(n,))# Input(shape=(n,))\n",
        "#input3 = tf.keras.layers.GaussianNoise(noise_std)(input2) #input1,stddev =0.5\n",
        "#x2 = tf.keras.layers.Dense(n, activation='relu')(input3)\n",
        "#Equivalent to subtracted = keras.layers.subtract([x1, x2])\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "#generator = tf.keras.models.Model(inputs=[input1, input2], outputs=out)\n",
        "generator = tf.keras.models.Model(inputs=[input1], outputs=out)\n",
        "generator.summary()\n",
        "#print(x.shape,(generator_noise(x)).shape)\n",
        "tf.print(generator([x]).shape)\n",
        "#test = generator(x)\n",
        "#print(test[1])\n",
        "generator.input\n",
        "#model.input"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "TensorShape([1000, 4])\n",
            "Model: \"model\"\n",
            "__________________________________________________________________________________________________\n",
            "Layer (type)                    Output Shape         Param #     Connected to                     \n",
            "==================================================================================================\n",
            "input_1 (InputLayer)            [(None, 4)]          0                                            \n",
            "__________________________________________________________________________________________________\n",
            "tf_op_layer_Shape (TensorFlowOp [(2,)]               0           input_1[0][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "tf_op_layer_strided_slice (Tens [()]                 0           tf_op_layer_Shape[0][0]          \n",
            "__________________________________________________________________________________________________\n",
            "tf_op_layer_random_normal/shape [(2,)]               0           tf_op_layer_strided_slice[0][0]  \n",
            "__________________________________________________________________________________________________\n",
            "tf_op_layer_RandomStandardNorma [(None, 4)]          0           tf_op_layer_random_normal/shape[0\n",
            "__________________________________________________________________________________________________\n",
            "tf_op_layer_mul (TensorFlowOpLa [(None, 4)]          0           tf_op_layer_RandomStandardNormal[\n",
            "__________________________________________________________________________________________________\n",
            "tf_op_layer_random_normal (Tens [(None, 4)]          0           tf_op_layer_mul[0][0]            \n",
            "__________________________________________________________________________________________________\n",
            "dense (Dense)                   (None, 4)            20          input_1[0][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "dense_1 (Dense)                 (None, 4)            20          tf_op_layer_random_normal[0][0]  \n",
            "__________________________________________________________________________________________________\n",
            "concatenate (Concatenate)       (None, 8)            0           dense[0][0]                      \n",
            "                                                                 dense_1[0][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "dense_2 (Dense)                 (None, 32)           288         concatenate[0][0]                \n",
            "__________________________________________________________________________________________________\n",
            "dense_3 (Dense)                 (None, 32)           1056        dense_2[0][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "dense_4 (Dense)                 (None, 4)            132         dense_3[0][0]                    \n",
            "==================================================================================================\n",
            "Total params: 1,516\n",
            "Trainable params: 1,516\n",
            "Non-trainable params: 0\n",
            "__________________________________________________________________________________________________\n",
            "TensorShape([1000, 4])\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor 'input_1:0' shape=(None, 4) dtype=float32>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 5
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "3CbjziKpv35v",
        "colab_type": "text"
      },
      "source": [
        "### Help Functions"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "I8rHD990Y-w8",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_EUzHiyUXLoP",
        "colab_type": "text"
      },
      "source": [
        "## Channels as Black-Box"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "W63_fJJRXL7A",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "\n",
        "\n",
        "def real_channel(x):\n",
        "    # Black-box Channel\n",
        "    #AWGN\n",
        "    return x + tf.random.normal(tf.shape(x), mean=0.0, stddev=noise_std)\n",
        "\n",
        "    #Rayleigh\n",
        "    #return x + tf.sqrt(tf.square(tf.random_normal(tf.shape(x), mean=0.0, stddev=noise_std)) + tf.square(tf.random_normal(tf.shape(x), mean=0.0, stddev=noise_std)))\n",
        "    \n",
        "    #Uniform U(-3;3)    \n",
        "    #return x + tf.random_uniform(tf.shape(x), minval=-2, maxval=2)\n",
        "\n",
        "    \n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Rzh-JZgfXSqN",
        "colab_type": "text"
      },
      "source": [
        "## Discriminator\n",
        "Model definition and creating discriminator\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "97h2eMLeXS68",
        "colab_type": "code",
        "outputId": "7b13f5ef-7fc5-456e-a39b-f167a1f44f9f",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 231
        }
      },
      "source": [
        "def concc(y,x):  \n",
        "  inputs = tf.concat(values=[y,x], axis=1)\n",
        "  return inputs\n",
        "\n",
        "def get_discriminator():\n",
        "  model = tf.keras.Sequential()\n",
        "  model.add(tf.keras.layers.Dense(32,use_bias=True, kernel_initializer=randN_initial,activation='relu',input_shape=((2*n,))))\n",
        "  #model.add(tf.keras.layers.Dense(32,use_bias=True, kernel_initializer=randN_initial, activation='relu'))\n",
        "  model.add(tf.keras.layers.Dense(1,use_bias=False, activation='sigmoid'))\n",
        "  return model\n",
        "discriminator = get_discriminator()\n",
        "\n",
        "discriminator.summary()\n",
        "\n",
        "#def discriminator(y,x):\n",
        "#    # Concatenate x and y\n",
        "#    inputs = tf.concat(values=[y,x], axis=1)\n",
        "#    #dense NN\n",
        "#    D_h1 = tf.nn.relu(tf.matmul(inputs, D_W1) + D_b1)\n",
        "#    D_logit = tf.matmul(D_h1, D_W2) + D_b2\n",
        "#    D_prob = tf.nn.sigmoid(D_logit)\n",
        "#    return D_prob, D_logit"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Model: \"sequential\"\n",
            "_________________________________________________________________\n",
            "Layer (type)                 Output Shape              Param #   \n",
            "=================================================================\n",
            "dense_5 (Dense)              (None, 32)                288       \n",
            "_________________________________________________________________\n",
            "dense_6 (Dense)              (None, 1)                 32        \n",
            "=================================================================\n",
            "Total params: 320\n",
            "Trainable params: 320\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "QRnlfRYuYC8R",
        "colab_type": "text"
      },
      "source": [
        "## Data Generation, überhaupt noch relevant??!!"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "EYcnkBIUXYa_",
        "colab_type": "text"
      },
      "source": [
        "## discriminator desicion????\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "F7im8FYMXeOV",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "cross_entropy = tf.keras.losses.BinaryCrossentropy(from_logits=True)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "m-xQt6M5Xd9P",
        "colab_type": "text"
      },
      "source": [
        "## Define Loss\n",
        "strongly inspiered by: \\\\\n",
        "https://www.tensorflow.org/beta/tutorials/generative/dcgan?hl=en"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "36yIH7Q3FiEq",
        "colab_type": "text"
      },
      "source": [
        "## defining Loss. TODO:\n",
        "compile the Model with the right loss functions"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "upCLjUsVDzAn",
        "colab_type": "code",
        "outputId": "218269e1-6568-4e10-80ca-7f1cc1885d55",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 285
        }
      },
      "source": [
        "x = tf.random.normal((batch_size,n),dtype=tf.dtypes.float32)    #randomly sample input data (\"fake\" AE messages)\n",
        "x = x/tf.sqrt(2*tf.reduce_mean(tf.square(x)))\n",
        "real_training_data = tf.concat(values=[real_channel(x), x], axis=-1)  \n",
        "fake_training_data = tf.concat(values=[generator([x]),x], axis=-1)# training =True),x], axis=-1)\n",
        "\n",
        "tf.print(real_training_data.shape,fake_training_data.shape)\n",
        "real_output = discriminator(real_training_data)\n",
        "fake_output = discriminator(fake_training_data)\n",
        "tf.print(fake_output)\n",
        "tf.print(real_output)\n",
        "#print(real_output, fake_output)"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "TensorShape([1000, 8]) TensorShape([1000, 8])\n",
            "[[0.505847812]\n",
            " [0.505140364]\n",
            " [0.494822353]\n",
            " ...\n",
            " [0.49722]\n",
            " [0.504154]\n",
            " [0.493402451]]\n",
            "[[0.490723401]\n",
            " [0.506768763]\n",
            " [0.485244304]\n",
            " ...\n",
            " [0.512139738]\n",
            " [0.469683796]\n",
            " [0.468602926]]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ERelQ5oTEMtO",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#def discriminator_loss(real_output, fake_output):\n",
        "  #loss= -tf.reduce_mean(tf.math.log(real_output) + tf.math.log(1. - fake_output))   #use \"-\" sign to minimize rather than maximize loss\n",
        "#  loss = tf.reduce_mean(real_output) - tf.reduce_mean(fake_output)  #Wasserstein GAN\n",
        "#  return loss\n",
        "\n",
        "def generator_loss(fake_output, generator):\n",
        "  return -tf.reduce_mean(fake_output)\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0VCZBgwYBNYW",
        "colab_type": "text"
      },
      "source": [
        "# Learning Rate"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "i8J9r3UpBNl1",
        "colab_type": "code",
        "outputId": "ab81b67e-4abc-4a8f-d8b6-e09176c72aa0",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "disc_loss = tf.reduce_mean(real_output) - tf.reduce_mean(fake_output)   #use \"-\" sign to minimize rather than maximize loss\n",
        "gen_loss =  -tf.reduce_mean(fake_output)\n",
        "#disc_loss = -tf.reduce_mean(tf.math.log(real_output) + tf.math.log(1. - fake_output)) #-tf.reduce_mean(tf.math.log(real_output) + tf.math.log(1. - fake_output))   #use \"-\" sign to minimize rather than maximize loss\n",
        "#gen_loss =-tf.reduce_mean(tf.math.log(fake_output))\n",
        "\n",
        "tf.print(disc_loss, gen_loss)\n",
        "\n",
        "generator_optimizer = tf.keras.optimizers.RMSprop(learning_rate=0.0001)      #RMSprop   in oreder to test where the error comes from\n",
        "discriminator_optimizer = tf.keras.optimizers.RMSprop(learning_rate=0.0001)      #"
      ],
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "-0.00844934583 -0.500320792\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "gktABNcepz5c",
        "colab_type": "text"
      },
      "source": [
        "# Evaluation with Histogram"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fgM9lv-dp1PI",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def generate_evaluation_data(batch_size=1000):\n",
        "  x = tf.random.normal((batch_size,n),dtype=tf.dtypes.float32)    #randomly sample input data (\"fake\" AE messages)\n",
        "  x = x/tf.sqrt(2*tf.reduce_mean(tf.square(x))) #Average power normalization (not required if standard normal distribution is used )\n",
        "  #G_n = tf.random.normal([tf.shape(x)[0],n],dtype=tf.float32) \n",
        "  #inputs = tf.concat(values=[x, G_n], axis=1)\n",
        "  fake_eval_data = tf.concat(values=[generator([x]), x], axis=1)\n",
        "  real_eval_data = tf.concat(values=[real_channel(x), x], axis=1) #tf.concat(values=[real_channel(x),x], axis=1)\n",
        "  inputs = x\n",
        "  return  real_eval_data, fake_eval_data, inputs \n",
        "\n",
        "\n",
        "\n",
        "def get_evaluation_data(evaluation_per_epochs):\n",
        "  real_eval_data = []\n",
        "  fake_eval_data  = []\n",
        "  inputs = []\n",
        "  for i in range(evaluation_per_epochs):\n",
        "    data = generate_evaluation_data()\n",
        "    real_eval_data.append(data[0])\n",
        "    fake_eval_data.append(data[1])\n",
        "    inputs.append(data[2])\n",
        "  return real_eval_data, fake_eval_data, inputs\n",
        "\n",
        "\n",
        "def test_eval(real_eval_data,fake_eval_data,inputs):\n",
        "  hist_range = 3\n",
        "  \n",
        "  #inputs_ = tf.concat(values=[inputs, inputs],  axis=0)\n",
        "  \n",
        "  fake_output_hist = np.mean(fake_eval_data,axis=0)  # Changed from 0 to 1\n",
        "  real_output_hist = np.mean(real_eval_data,axis=0)\n",
        "  inputs_hist = np.mean(inputs,axis=0)\n",
        "    \n",
        "  fake_output_hist1 = np.reshape( fake_output_hist,[-1,])\n",
        "  real_output_hist1 = np.reshape( real_output_hist,[-1,])\n",
        "    \n",
        "  plt.hist(fake_output_hist1,bins=100,range=(-hist_range,hist_range),density=True,histtype='step')\n",
        "  plt.hist(real_output_hist1,bins=100,range=(-hist_range,hist_range),density=True,histtype='step')    \n",
        "  plt.title(\"noise distribution\")\n",
        "  plt.legend([\"generator\", \"target\"])\n",
        "  plt.show()\n",
        "  tf.print(inputs_hist.shape)\n",
        "  \n",
        "  #fake_noise = np.reshape( fake_output_hist - inputs_hist,[-1,])\n",
        "  #real_noise = np.reshape( real_output_hist - inputs_hist,[-1,])\n",
        "   \n",
        "  #plt.hist(fake_noise,bins=300,range=(-hist_range,hist_range),density=True,histtype='step')\n",
        "  #plt.hist(real_noise,bins=300,range=(-hist_range,hist_range),density=True,histtype='step')    \n",
        "  #plt.title(\"noise distribution after subtracting Inpus_noise\")\n",
        "  #plt.legend([\"generator\", \"target\"])\n",
        "  #plt.show()\n",
        "    \n",
        "    #print(\"decision for fake data was %d: and for real data was %d:\" % (decision_fake, decision_real))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "RXQWOgXnl62o",
        "colab_type": "text"
      },
      "source": [
        "### Define the training loop"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1sl75gEZl6Rv",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "epochs = 2000\n",
        "steps_per_epoches = 100\n",
        "batch_size = 1000\n",
        "\n",
        "evaluation_per_epochs = 10\n",
        "\n",
        "seed = tf.random.normal([batch_size, n])\n",
        "\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "AZUTZng_fFBk",
        "colab_type": "text"
      },
      "source": [
        "# Wasserstein clipping"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GEFEdt29fEj0",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#clip_D = [p.assign(tf.clip_by_value(p, -0.001, 0.001)) for p in discriminator.trainable_variables]\n",
        "\n",
        "#def get_disc_grad(trainable_variables):\n",
        "#  return [p.assign(tf.clip_by_value(p, -0.01, 0.01)) for p in trainable_variables]\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ooDukkHvmduJ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def train(epochs, steps_per_epoches , batch_size):\n",
        "  start = time.time()\n",
        "  counter = 0\n",
        "  epoch = 0\n",
        "  for epoch in range(epochs):\n",
        "    #print(massege_batch)\n",
        "    counter += 1\n",
        "\n",
        "    \n",
        "    train_step() \n",
        "    #tf.print(generator_optimizer.apply_gradients())\n",
        "    #if counter%5 == 0:\n",
        "    if counter%100==0:\n",
        "      tf.print(\"counter %d:\" % (counter))\n",
        "      fake_c = generator([x,generator_noise(x)])\n",
        "      tf.print(fake_c[0])\n",
        "    if counter%6 == 0 and counter<8:\n",
        "      real_eval_data, fake_eval_data, inputs = get_evaluation_data(evaluation_per_epochs)\n",
        "      test_eval(real_eval_data, fake_eval_data, inputs)\n",
        "    if counter%1000 == 0:\n",
        "      real_c = real_channel(x)\n",
        "      fake_c = generator([x,generator_noise(x)])\n",
        "      real_eval_data, fake_eval_data, inputs = get_evaluation_data(evaluation_per_epochs)\n",
        "      test_eval(real_eval_data, fake_eval_data, inputs)\n",
        "      tf.print(fake_c[0])\n",
        "      tf.print(disc_loss, gen_loss)\n",
        "    #print ('Time for epoch {} is {} sec,'.format(epoch + 1, time.time()-start))\n",
        "      tf.print ('Time for epoch {},'.format(epoch + 1))\n",
        "    x = tf.random.normal((batch_size,n),dtype=tf.dtypes.float32)    #randomly sample input data (\"fake\" AE messages)\n",
        "    x = x/tf.sqrt(2*tf.reduce_mean(tf.square(x)))\n",
        "    #print(x)\n",
        "    fake_c = generator(x)\n",
        "    if tf.math.is_nan(fake_c[0,0]) == True:\n",
        "      print(\"doesn't train the generator as expacted\")\n",
        "      tf.debugging.check_numerics(fake_c,'message generator',name=None)\n",
        "      break # in order to finde wher the [nan] - prolem is cumming from\n",
        "    \n",
        "       \n",
        "  #checkpoint_path = \"training_1/cp.ckpt\"\n",
        "  #checkpoint_dir = os.path.dirname(checkpoint_path)\n",
        "\n",
        "  #cp_callback = tf.keras.callbacks.ModelCheckpoint(checkpoint_path,\n",
        "  #                                               save_weights_only=False,\n",
        "  #                                               verbose=1)    \n",
        "  tf.saved_model.save(generator,'/tmp/saved_model/')\n",
        "  tf.print ('Time for the training is {} sec,'.format( time.time()-start))\n",
        " # print(gradients_of_generator)  \n",
        "  \n",
        "\n",
        "  # Generate after the final epoch\n",
        "    "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "j7H98i7TmVxw",
        "colab_type": "text"
      },
      "source": [
        "\n",
        "The training loop begins with generator receiving a random seed as input. That seed is used to produce an image. The discriminator is then used to classify real images (drawn from the training set) and fakes images (produced by the generator). The loss is calculated for each of these models, and the gradients are used to update the generator and discriminator."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0XxSryMYmCkH",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#@tf.function\n",
        "#def train_step(massege_batch,counter):\n",
        "#    x = tf.random.normal((batch_size,n),dtype=tf.dtypes.float32)    #randomly sample input data (\"fake\" AE messages)\n",
        "#    x = x/tf.sqrt(2*tf.reduce_mean(tf.square(x)))\n",
        "    #G_n = tf.random.normal([tf.shape(x)[0],n],dtype=tf.float32) \n",
        "    #inputs = tf.concat(values=[x, G_n], axis=1)\n",
        "#    real_training_data = tf.concat(values=[real_channel(x), x], axis=1)  #tf.concat(values=[real_channel(x),x], axis=1)\n",
        "\n",
        "\n",
        " #   with tf.GradientTape() as gen_tape, tf.GradientTape() as disc_tape:           #tapes the gradient of the generaor an the discriminator\n",
        "  #    fake_training_data = tf.concat(values=[generator(x, training =True),x], axis=1)\n",
        "      \n",
        " #     real_output = discriminator(real_training_data, training=True)\n",
        " #     fake_output = discriminator(fake_training_data, training=True)\n",
        "\n",
        " #     disc_loss = -tf.reduce_mean(tf.math.log(real_output) + tf.math.log(1. - fake_output))   #use \"-\" sign to minimize rather than maximize loss\n",
        " #     gen_loss = -tf.reduce_mean(tf.math.log(fake_output))\n",
        "\n",
        " #     gradients_of_generator = gen_tape.gradient(gen_loss, generator.trainable_variables)\n",
        " #     gradients_of_discriminator = disc_tape.gradient(disc_loss, discriminator.trainable_variables)\n",
        "\n",
        "  #    generator_optimizer.apply_gradients(zip(gradients_of_generator, generator.trainable_variables))\n",
        "  #    discriminator_optimizer.apply_gradients(zip(gradients_of_discriminator, discriminator.trainable_variables))\n",
        "    \n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bJno--QQh4_w",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "@tf.function\n",
        "def train_step(): #epoch, steps_per_epoches , batch_size, generator, discriminator):\n",
        "  x = tf.random.normal((batch_size,n),dtype=tf.dtypes.float32)    #randomly sample input data (\"fake\" AE messages)\n",
        "  x = x/tf.sqrt(2*tf.reduce_mean(tf.square(x)))\n",
        "  for i in range(5):\n",
        "    with tf.GradientTape() as gen_tape, tf.GradientTape() as disc_tape:\n",
        "      real_training_data = tf.concat(values=[real_channel(x), x], axis=1)\n",
        "      fake_training_data = tf.concat(values=[generator(x),x], axis=1)# training =True),x], axis=1)\n",
        "      real_output = discriminator(real_training_data)#, training=True)\n",
        "      fake_output = discriminator(fake_training_data)\n",
        "      \n",
        "      \n",
        "      #disc_loss = tf.reduce_mean(real_output) - tf.reduce_mean(fake_output)  #use \"-\" sign to minimize rather than maximize loss\n",
        "      #gen_loss =  -tf.reduce_mean(fake_output)\n",
        "      #tf.print(real_training_data.shape, real_output[1].shape)\n",
        "      #tf.debugging.check_numerics(disc_loss,'loss generator',name=None)\n",
        "      # clip_D = [p.assign(tf.clip_by_value(p, -0.01, 0.01)) for p in gradients_of_discriminator]   \n",
        "      #tf.print(real_training_data[0])\n",
        "      \n",
        "      disc_loss = -tf.reduce_mean(tf.math.log(real_output) + tf.math.log(1. - fake_output)) #-tf.reduce_mean(tf.math.log(real_output) + tf.math.log(1. - fake_output))   #use \"-\" sign to minimize rather than maximize loss\n",
        "      gen_loss =-tf.reduce_mean(tf.math.log(fake_output))\n",
        "      #disc_loss = tf.reduce_mean(real_output) - tf.reduce_mean(fake_output)   #use \"-\" sign to minimize rather than maximize loss\n",
        "      #gen_loss =  -tf.reduce_mean(fake_output)\n",
        "      #tf.print(disc_loss, gen_loss)\n",
        "      if tf.math.is_nan(disc_loss) == False:\n",
        "        gradients_of_discriminator = disc_tape.gradient(disc_loss, discriminator.trainable_variables)\n",
        "        discriminator_optimizer.apply_gradients(zip(gradients_of_discriminator, discriminator.trainable_variables))\n",
        "        #tf.print(disc_loss, gen_loss)\n",
        "      if i == 4:  \n",
        "        gradients_of_generator = gen_tape.gradient(gen_loss, generator.trainable_variables)\n",
        "        generator_optimizer.apply_gradients(zip(gradients_of_generator, generator.trainable_variables))\n",
        "        #tf.print(disc_loss, gen_loss)\n",
        "\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "UuGMDjc1metC",
        "colab_type": "text"
      },
      "source": [
        "## Train the model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "y82FQj3Jmvxx",
        "colab_type": "code",
        "outputId": "dc18e9ec-750f-4077-94f3-087933442a26",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "%%time\n",
        "train(epochs, steps_per_epoches , batch_size)\n",
        "print(generator(x)[1])\n",
        "generator.summary()\n",
        "discriminator.summary()\n",
        "\n"
      ],
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX4AAAEICAYAAABYoZ8gAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAH5tJREFUeJzt3X98VfWd5/HXW4pEBYMC7VQgQFvr\nL35oDWjFijgt0jKKneoWtVOdbZtq6+o47UzRmS2s2tl09DF11XaVbXm0XRW1dWyx4vhj/TVVqYCL\niuAP1CiJrvyQgFRwCHz2j3uSHuK95Ca5yc3NeT8fjzw4v8/n3Oj73nzP936PIgIzM8uOfcpdgJmZ\n9S4Hv5lZxjj4zcwyxsFvZpYxDn4zs4xx8JuZZYyD3/o0SdskfayHz/GIpK8n0+dKur+Ex35e0snJ\n9HxJN5fw2JdL+mmpjmfZ8aFyF2C2NxExuJfPdwtwS0fbSfo50BgR/9jB8Y4qRV3Jm8fNETEqdex/\nKsWxLXv8id+sB0jyhyrrsxz81uMkNUj6rqRnJW2RdLukqtT6b0haK+kdSYslHZJaF5I+kUx/QdJq\nSe9KapL03dR2fyFppaRmSU9ImriXej4n6YWklhsApdadL+n3ybQk/UjSeklbJT0nabykOuBc4O+T\npqi7U9f5PUnPAn+U9KFk2WdTp69Krv9dSU9LmpTvWpP5n0u6StIBwL3AIcn5tkk6pH3TkaTTk6al\n5qT56ohifweWLQ5+6y3/CZgJjAMmAucDSDoF+O/J+o8CrwO3FTjGz4BvRsQQYDzwUHKMY4CFwDeB\nYcBNwGJJg9ofQNJw4F+BfwSGA68AUwucbwZwEvBJoDqpcVNELCDXHPTPETE4Ik5L7XM2MAsYGhEt\neY45G/gVcDBwK/AbSQMLnB+AiPgj8HngzeR8gyPizXbX9UlgEfA3wAhgCXC3pH1Tm+X9HVj2OPit\nt1wXEW9GxDvA3cDRyfJzgYUR8XREvA9cBnxa0tg8x9gJHCnpwIjYHBFPJ8vrgJsi4g8RsSsifgG8\nDxyf5xhfAJ6PiF9HxE7gWuD/Fah5JzAEOBxQRKyJiLeKuM51EbG9wPoVqXP/C1BVoM7O+jJwT0Q8\nkBz7GmA/4IR2teX7HVjGOPitt6TD9T2g9abtIeQ+5QMQEduATcDIPMf4Erngfl3So5I+nSwfA3wn\naeJoltQMjE6O3d4hwLrU+SI9nxYRDwE3AD8G1ktaIOnADq4z77HyrY+I3UBjgTo7q/3ruDs5V/p1\nLPQ7sIxx8Fu5vUkuuAFI2rOHAU3tN4yIZRExG/gw8BvgjmTVOuAHETE09bN/RCzKc763yL0ptJ5P\n6fk857wuIo4FjiTX5PN3rasK7VLoWIn0ufcBRpF7DSAXxvuntv2zThy3/evYel0feB3NHPxWbouA\nv5Z0dNIm/0/AHyKiIb2RpH2TPvbVSVPGVmB3svp/ARdIOi65IXuApFmShuQ53z3AUZL+Mul5czF7\nBmz6nJOTYw4E/gjsSJ3zbaAr3y84NnXuvyHXJLU0WbcSOEfSAEkzgWmp/d4GhkmqLnDcO4BZkv48\nqfc7ybGf6EKN1s85+K2sIuJB4L8Cd5L7NP5xYE6Bzf8KaJC0FbiA3P0BImI58A1yzTKbgbUUuHEZ\nERuBs4B6ck1KhwKPFzjfgeTeVDaTa0bZBFydrPsZufsNzZJ+U9zVAvBbcu3xm5Pr+cvkjQzgEuA0\noDm5trbjRsQL5N4kX03OuUfzUES8CHwFuB7YmBzntIj4j07UZhkhP4jFzCxb/InfzCxjHPxmZhnj\n4DczyxgHv5lZxvTJgaSGDx8eY8eOLXcZZmYVY8WKFRsjYkQx2/bJ4B87dizLly8vdxlmZhVD0usd\nb5Xjph4zs4xx8JuZZYyD38wsY/pkG7+Z9R87d+6ksbGRHTt2lLuUfqGqqopRo0YxcOBeH+OwVw5+\nM+tRjY2NDBkyhLFjx5IbNNS6KiLYtGkTjY2NjBs3rsvHcVOPmfWoHTt2MGzYMId+CUhi2LBh3f7r\nycFvZj3OoV86pXgtHfxmZhnTYRu/pIXAXwDrI2J8nvV/RzIuenK8I4AREfGOpAbgXWAX0BIRtaUq\n3Mwq09T6h2hqLvRI4s4bOXQ/Hp97SsmO19OuvfZa6urq2H///TveuIcUc3P35+QecPHLfCsj4mqS\nh1NIOg24NHmYc6vpycMvzCrfjybAljdy09U1cOlz5a2nAjU1b6ehflbJjjd27j0lO1YpRAQRwT77\n5G9Qufbaa/nKV77SqeDftWsXAwYMKFWJHTf1RMRjwDsdbZc4m9xTgsz6py1vwPwtuZ/WNwCrCFde\neSWHHXYYJ554ImeffTbXXHMNr7zyCjNnzuTYY4/lM5/5DC+88AIA559/PhdffDEnnHACH/vYx/j1\nr3/ddpyrr76ayZMnM3HiRObNmwdAQ0MDhx12GF/96lcZP34869at48ILL6S2tpajjjqqbbvrrruO\nN998k+nTpzN9+nQAFi1axIQJExg/fjzf+9732s4zePBgvvOd7zBp0iSefPLJ0r4Yre9Oe/sBxgKr\nOthmf3JvEAenlr0GPA2sAOqKOVdEcOyxx4ZZnzTvwPzTVtDq1av3mB/zvd+V9PjFHO+pp56KSZMm\nxfbt22Pr1q3xiU98Iq6++uo45ZRT4qWXXoqIiKVLl8b06dMjIuK8886LM888M3bt2hXPP/98fPzj\nH4+IiPvuuy++8Y1vxO7du2PXrl0xa9asePTRR+O1114LSfHkk0+2nXPTpk0REdHS0hLTpk2LZ555\nJlfvmDGxYcOGiIhoamqK0aNHx/r162Pnzp0xffr0uOuuuyIiAojbb7897/W0f02T7ZdHkRlbyn78\npwGPx57NPCdGRJOkDwMPSHohcn9BfICkOqAOoKampoRlmVnWPf7448yePZuqqiqqqqo47bTT2LFj\nB0888QRnnXVW23bvv/9+2/QZZ5zBPvvsw5FHHsnbb78NwP3338/999/PMcccA8C2bdt4+eWXqamp\nYcyYMRx//PFt+99xxx0sWLCAlpYW3nrrLVavXs3EiRP3qGvZsmWcfPLJjBiRG1Tz3HPP5bHHHuOM\nM85gwIABfOlLX+qR16OUwT+Hds08EdGU/Lte0l3AFCBv8EfEAmABQG1trR8EbGY9avfu3QwdOpSV\nK1fmXT9o0KC26UieTR4RXHbZZXzzm9/cY9uGhgYOOOCAtvnXXnuNa665hmXLlnHQQQdx/vnnd7rv\nfVVVVUnb9dNK0p1TUjUwDfhtatkBkoa0TgMzgFWlOJ+ZWWdMnTqVu+++mx07drBt2zZ+97vfsf/+\n+zNu3Dh+9atfAblQf+aZZ/Z6nFNPPZWFCxeybds2AJqamli/fv0Httu6dSsHHHAA1dXVvP3229x7\n771t64YMGcK7774LwJQpU3j00UfZuHEju3btYtGiRUybNq1Ul11QMd05FwEnA8MlNQLzgIEAEXFj\nstkXgfsj4o+pXT8C3JV82eBDwK0R8W+lK93MKtHIofuVtCfOyKH7dbjN5MmTOf3005k4cSIf+chH\nmDBhAtXV1dxyyy1ceOGFXHXVVezcuZM5c+YwadKkgseZMWMGa9as4dOf/jSQuwF78803f+CT+aRJ\nkzjmmGM4/PDDGT16NFOnTm1bV1dXx8yZMznkkEN4+OGHqa+vZ/r06UQEs2bNYvbs2V18JYqn1j9h\n+pLa2trwg1isT5pfnevR037aClqzZg1HHHFEuctg27ZtDB48mPfee4+TTjqJBQsW8KlPfarcZXVJ\nvtdU0ooo8rtSHqTNzDKhrq6O1atXs2PHDs4777yKDf1ScPCbdaT9l7asIt16663lLqHPcPCbdaT1\nS1tm/YQHaTMzyxgHv5lZxjj4zcwyxm38Zta70jfLS6GDUVKbm5u59dZb+da3vlW6c+bxyCOPsO++\n+3LCCSf06HlKwcFv1lXVNbm+/K3THqK5OKW+Wd76OyigubmZn/zkJ0UHf+tAZoWGVS7kkUceYfDg\nwQ5+s34tHfQdhI+Vz9y5c3nllVc4+uijmT59Os8++yybN29m586dXHXVVcyePZuGhgZOPfVUjjvu\nOFasWMGSJUt48MEH+eEPf8jQoUOZNGkSgwYN4oYbbmDDhg1ccMEFvPFG7q+Wa6+9lpEjR3LjjTcy\nYMAAbr75Zq6//no+85nPlPnKC3Pwm1m/Vl9fz6pVq1i5ciUtLS289957HHjggWzcuJHjjz+e008/\nHYCXX36ZX/ziFxx//PG8+eabXHnllTz99NMMGTKEU045pW0oh0suuYRLL72UE088kTfeeINTTz2V\nNWvWcMEFFzB48GC++93vlvNyi+LgN7PMiAguv/xyHnvsMfbZZx+amprahlxOD6v81FNPMW3aNA4+\n+GAAzjrrLF566SUAHnzwQVavXt12zK1bt7YN2lYpHPxmlhm33HILGzZsYMWKFQwcOJCxY8e2DZec\nHlZ5b3bv3s3SpUupqqrqyVJ7lLtzmlm/lh4GecuWLXz4wx9m4MCBPPzww7z++ut595k8eTKPPvoo\nmzdvpqWlhTvvvLNt3YwZM7j++uvb5lvH80+fp6/zJ34z613p3lClOt5eDBs2jKlTpzJ+/HgmT57M\nCy+8wIQJE6itreXwww/Pu8/IkSO5/PLLmTJlCgcffDCHH3441dW5mq+77jq+/e1vM3HiRFpaWjjp\npJO48cYbOe200zjzzDP57W9/65u7ZmZ7KEO312IGaFu1as/nRJ1zzjnU1dXR0tLCF7/4Rc444wwA\nhg8fzu233/6B/T/5yU/y7LPPlqbgHubgN8vHI3Jm3vz583nwwQfZsWMHM2bMaAv+/sDBb5aPR+TM\nvGuuuabcJfQY39w1sx7XF5/0V6lK8Vo6+M2sR1VVVbFp0yaHfwlEBJs2bep2V1I39ZhZjxo1ahSN\njY1s2LCh3KX0C1VVVYwaNapbx3Dwm1mPGjhwIOPGjSt3GZbSYVOPpIWS1ktaVWD9yZK2SFqZ/Hw/\ntW6mpBclrZU0t5SFm5lZ1xTTxv9zYGYH2/x7RByd/FwBIGkA8GPg88CRwNmSjuxOsWZm1n0dBn9E\nPAa804VjTwHWRsSrEfEfwG3A7C4cx8zMSqhUvXo+LekZSfdKOipZNhJYl9qmMVmWl6Q6ScslLfdN\nIDOznlOK4H8aGBMRk4Drgd905SARsSAiaiOidsSIESUoy8zM8ul28EfE1ojYlkwvAQZKGg40AaNT\nm45KlpmZWRl1O/gl/ZkkJdNTkmNuApYBh0oaJ2lfYA6wuLvnMzOz7umwH7+kRcDJwHBJjcA8YCBA\nRNwInAlcKKkF2A7MidxX9FokXQTcBwwAFkbE8z1yFWZmVrQOgz8izu5g/Q3ADQXWLQGWdK00MzPr\nCR6rx8wsYxz8ZmYZ47F6zFr54SuWEQ5+s1bdefhK+jmy1TVlebygWbEc/GalkA76Uj5I3KwHuI3f\nzCxjHPxmZhnjph6zTpha/xBNzdsBGDl0Px6fe0qZKzLrPAe/WSc0NW+noX4WkHsTGDv3HsBvAlZZ\nHPxmXZQO+tY3ALNK4DZ+M7OMcfCbmWWMg9/MLGMc/GZmGePgNzPLGAe/mVnGOPjNzDLGwW9mljEO\nfjOzjHHwm5llTIfBL2mhpPWSVhVYf66kZyU9J+kJSZNS6xqS5SslLS9l4WZm1jXFfOL/OTBzL+tf\nA6ZFxATgSmBBu/XTI+LoiKjtWolmZlZKHQ7SFhGPSRq7l/VPpGaXAqO6X5ZZ70gPs9xQVeZizHpJ\nqUfn/Bpwb2o+gPslBXBTRLT/a6CNpDqgDqCmxg+6tt6RHmaZ+WUtxazXlCz4JU0nF/wnphafGBFN\nkj4MPCDphYh4LN/+yZvCAoDa2tooVV1mZrankgS/pInAT4HPR8Sm1uUR0ZT8u17SXcAUIG/wm/UF\nfrCKZUG3u3NKqgH+FfiriHgptfwASUNap4EZQN6eQWZ9RUP9LBrqZ7W1+5v1Rx1+4pe0CDgZGC6p\nEZgHDASIiBuB7wPDgJ9IAmhJevB8BLgrWfYh4NaI+LceuAYzM+uEYnr1nN3B+q8DX8+z/FVg0gf3\nMDOzcvIzd806kO7yOXLofmWuxqz7HPxmHdijy6dZP+CxeszMMsbBb2aWMQ5+M7OMcfCbmWWMg9/M\nLGMc/GZmGePgNzPLGPfjN8tj5ND99hiwzaw/cfCb5dGtkTmra2B+9Z+mL32uNEWZlYiD36zU0kHf\n+gZg1oe4jd/MLGMc/GZmGeOmHsu03w+6GOafk5up9rOeLRsc/JZpo7QR5m8pdxlmvcrBb1YC7bt/\n+nm91pc5+M1KIB30rW8AZn2Vb+6amWWMg9/MLGOKCn5JCyWtl7SqwHpJuk7SWknPSvpUat15kl5O\nfs4rVeFmZtY1xX7i/zkwcy/rPw8cmvzUAf8TQNLBwDzgOGAKME/SQV0t1szMuq+o4I+Ix4B39rLJ\nbOCXkbMUGCrpo8CpwAMR8U5EbAYeYO9vIGZm1sNK1cY/EliXmm9MlhVa/gGS6iQtl7R8w4YNJSrL\nzMza6zM3dyNiQUTURkTtiBEjyl2OmVm/VargbwJGp+ZHJcsKLTczszIpVfAvBr6a9O45HtgSEW8B\n9wEzJB2U3NSdkSwzM7MyKeqbu5IWAScDwyU1kuupMxAgIm4ElgBfANYC7wF/nax7R9KVwLLkUFdE\nxN5uEpuZWQ8rKvgj4uwO1gfw7QLrFgILO1+amZn1hD5zc9fMzHqHg9/MLGMc/GZmGePgNzPLGAe/\nmVnGOPjNzDLGwW9mljEOfjOzjHHwm5lljB+2btaTqmtgfvWfpi99rrz1mOHgtwyaWv8QTc3bAWio\n6uGTpYO+9Q3ArMwc/JY5Tc3baaiflZuZX9ZSzMrCbfxmZhnj4DczyxgHv5lZxjj4zcwyxsFvZpYx\nDn4zs4xx8JuZZYyD38wsY4oKfkkzJb0oaa2kuXnW/0jSyuTnJUnNqXW7UusWl7J4MzPrvA6/uStp\nAPBj4HNAI7BM0uKIWN26TURcmtr+vwDHpA6xPSKOLl3JZt3z+0EXw/xzcjPVNeUtxqwMihmyYQqw\nNiJeBZB0GzAbWF1g+7OBeaUpz6z0RmkjzN/SY8cfOXQ/xs69p2368bmn9Ni5zLqimOAfCaxLzTcC\nx+XbUNIYYBzwUGpxlaTlQAtQHxG/KbBvHVAHUFPjT2FWudJB3/oGYNaXlPrm7hzg1xGxK7VsTETU\nAucA10r6eL4dI2JBRNRGRO2IESNKXJaZmbUqJvibgNGp+VHJsnzmAIvSCyKiKfn3VeAR9mz/NzOz\nXlZM8C8DDpU0TtK+5ML9A71zJB0OHAQ8mVp2kKRByfRwYCqF7w2YmVkv6LCNPyJaJF0E3AcMABZG\nxPOSrgCWR0Trm8Ac4LaIiNTuRwA3SdpN7k2mPt0byMzMel9RD2KJiCXAknbLvt9ufn6e/Z4AJnSj\nPjMzKzF/c9fMLGMc/GZmGePgNzPLGAe/mVnGOPjNzDLGwW9mljFFdec0q3g/mgBb3gCgMYYzqhw1\nVNfA/Oo/TV/6XDmqMHPwW0ZseaNtRM4T595DQzlqSAd96xuAWRm4qcfMLGP8id+sTDxmv5WLg98q\n3tT6h2hq3g5UVog21M8CPGa/9T4Hv1W8pubtDlGzTnAbv5lZxjj4zcwyxsFvZpYxDn4zs4zxzV2z\nXpLufdRQVeZiLNMc/Ga9JN37iPllLcUyzsFvmZH+wpRZljn4LTPaPm2bZVxRN3clzZT0oqS1kubm\nWX++pA2SViY/X0+tO0/Sy8nPeaUs3qyvGzl0P8bOvYexc+/xXxrWZ3T4iV/SAODHwOeARmCZpMUR\nsbrdprdHxEXt9j0YmAfUAgGsSPbdXJLqzfq4YoaPaH1zaJ2ulCEnrHIV09QzBVgbEa8CSLoNmA20\nD/58TgUeiIh3kn0fAGYCi7pWrtneVWKIpmv0kBPWG4oJ/pHAutR8I3Bcnu2+JOkk4CXg0ohYV2Df\nkflOIqkOqAOoqakpoiyzD3KImnWsVF/guhsYGxETgQeAX3T2ABGxICJqI6J2xIgRJSrLzMzaKyb4\nm4DRqflRybI2EbEpIt5PZn8KHFvsvmZm1ruKCf5lwKGSxknaF5gDLE5vIOmjqdnTgTXJ9H3ADEkH\nSToImJEsM+txvx90ce4Rh/OraYzh5S7HrM/osI0/IlokXUQusAcACyPieUlXAMsjYjFwsaTTgRbg\nHeD8ZN93JF1J7s0D4IrWG71mPW2UNpb/ObtmfVBRX+CKiCXAknbLvp+avgy4rMC+C4GF3ajRrMv8\nbV2zD/I3d60itX/cYiH+tq7ZBzn4rSLtMeCZmXWKx+M3M8sYf+I360Mq8ZvHVnkc/GblUF2T62ra\nOn3pc4C/eWy9w8FvVg5J0AN/egMw6yVu4zczyxgHv5lZxjj4zcwyxsFvZpYxDn4zs4xx8JuZZYyD\n38wsYxz8ZmYZ4+A3M8sYB7+ZWcZ4yAarCOnx98EPVjHrDge/VYQsjr/vkTqtpzj4zfooj9RpPcVt\n/GZmGVNU8EuaKelFSWslzc2z/m8lrZb0rKT/I2lMat0uSSuTn8WlLN7MzDqvw6YeSQOAHwOfAxqB\nZZIWR8Tq1Gb/F6iNiPckXQj8M/DlZN32iDi6xHWbmVkXFdPGPwVYGxGvAki6DZgNtAV/RDyc2n4p\n8JVSFmlWtB9NgC1v5Kara8pbi1kfVUzwjwTWpeYbgeP2sv3XgHtT81WSlgMtQH1E/KbTVZoVa8sb\nMH9LuavonAKPYTTrKSXt1SPpK0AtMC21eExENEn6GPCQpOci4pU8+9YBdQA1Nf6kZhnixzBaLyvm\n5m4TMDo1PypZtgdJnwX+ATg9It5vXR4RTcm/rwKPAMfkO0lELIiI2oioHTFiRNEXYGZmnVNM8C8D\nDpU0TtK+wBxgj945ko4BbiIX+utTyw+SNCiZHg5MJXVvwMzMel+HTT0R0SLpIuA+YACwMCKel3QF\nsDwiFgNXA4OBX0kCeCMiTgeOAG6StJvcm0x9u95AZmbWy4pq44+IJcCSdsu+n5r+bIH9ngAmdKdA\nMzMrLX9z18wsYxz8ZmYZ40HazCqAR+q0UnLwm1UAj9RppeSmHjOzjHHwm5lljJt6zPoSj9tjvcDB\nb9aXeNwe6wVu6jEzyxh/4rc+a2r9QzQ1bwdyXRgL8hj8Zp3i4Lc+q6l5Ow31szresBLH4DcrIzf1\nmJlljIPfzCxj3NRjlcnt+mZd5uC3ypSFdv0Cffo9bo91l4PfrK8q0Kff4/ZYd7mN38wsY/yJ3/qU\novvum1mXOfitTym6776ZdZmD3ypDuhcPZK8njwdvsxJy8FtlyEIvnr3x4G1WQkUFv6SZwP8ABgA/\njYj6dusHAb8EjgU2AV+OiIZk3WXA14BdwMURcV/Jqrd+we36nZT69P9k1QjGzv3TKnfvtGJ0GPyS\nBgA/Bj4HNALLJC2OiNWpzb4GbI6IT0iaA/wQ+LKkI4E5wFHAIcCDkj4ZEbtKfSFWuQq26/tLWvml\nPv1/dH71Hq/d1PqH3MffOlTMJ/4pwNqIeBVA0m3AbCAd/LOB+cn0r4EbJClZfltEvA+8Jmltcrwn\nS1O+VZL0J/u0J6sugfnnfHCH6ppsN+8UI932DzxeXQP1uTcG9/G3QooJ/pHAutR8I3BcoW0iokXS\nFmBYsnxpu31H5juJpDqgLpndJunFImrLZziwsYv79jX95Vr2eh2HFNxtFfyteqKe7ujjv5M9XzP9\ncK8b9/FrKVp/uQ7o3rWMKXbDPnNzNyIWAAu6exxJyyOitgQllV1/uZb+ch3ga+mL+st1QO9dSzHf\n3G0CRqfmRyXL8m4j6UNANbmbvMXsa2ZmvaiY4F8GHCppnKR9yd2sXdxum8XAecn0mcBDERHJ8jmS\nBkkaBxwKPFWa0s3MrCs6bOpJ2uwvAu4j151zYUQ8L+kKYHlELAZ+Bvzv5ObtO+TeHEi2u4PcjeAW\n4Nu90KOn281FfUh/uZb+ch3ga+mL+st1QC9di3IfzM3MLCs8OqeZWcY4+M3MMqZfBr+kKyU9K2ml\npPslFe4q3odJulrSC8m13CVpaLlr6ipJZ0l6XtJuSRXX9U7STEkvSloraW7He/RdkhZKWi9pVblr\n6Q5JoyU9LGl18t/WJeWuqaskVUl6StIzybX8tx49X39s45d0YERsTaYvBo6MiAvKXFanSZpBrodU\ni5T7Kk5EfK/MZXWJpCOA3cBNwHcjYnmZSypaMmzJS6SGLQHObjdsScWQdBKwDfhlRIwvdz1dJemj\nwEcj4mlJQ4AVwBmV+HtJRjo4ICK2SRoI/B64JCKWdrBrl/TLT/ytoZ84AKjId7eIuD8iWpLZpeS+\nB1GRImJNRHT129jl1jZsSUT8B9A6bElFiojHyPW+q2gR8VZEPJ1MvwusocDIAH1d5GxLZgcmPz2W\nW/0y+AEk/UDSOuBc4PvlrqcE/jNwb7mLyKh8w5ZUZMD0V5LGAscAfyhvJV0naYCklcB64IGI6LFr\nqdjgl/SgpFV5fmYDRMQ/RMRo4BbgovJWW1hH15Fs8w/kvgdxS/kq7Vgx12JWapIGA3cCf9Pur/2K\nEhG7IuJocn/ZT5HUY81wfWasns6KiM8WuektwBJgXg+W02UdXYek84G/AP48+vgNmU78TiqNhx7p\no5L28DuBWyLiX8tdTylERLOkh4GZQI/cgK/YT/x7I+nQ1Oxs4IVy1dIdyQNw/h44PSLeK3c9GVbM\nsCXWy5Iboj8D1kTEv5S7nu6QNKK1156k/ch1JOix3OqvvXruBA4j14vkdeCCiKi4T2jJEBiDyA14\nB7C0EnsnAUj6InA9MAJoBlZGxKnlrap4kr4AXMufhi35QZlL6jJJi4CTyQ0B/DYwLyJ+VtaiukDS\nicC/A8+R+38d4PKIWFK+qrpG0kTgF+T++9oHuCMiruix8/XH4Dczs8L6ZVOPmZkV5uA3M8sYB7+Z\nWcY4+M3MMsbBb2aWMQ5+M7OMcfCbmWXM/wcH+msKZ71L2AAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "(1000, 4)\n",
            "counter 100:\n",
            "[0.342396796 -0.67822516 0.532614768 -0.37942329]\n",
            "counter 200:\n",
            "[-0.0978322178 -0.0844742954 0.00549814757 -0.436805069]\n",
            "counter 300:\n",
            "[-0.133961156 0.0710996389 -0.168605864 -0.0108040795]\n",
            "counter 400:\n",
            "[-0.515576184 -0.233261287 0.770923197 -0.0794517547]\n",
            "counter 500:\n",
            "[-0.234608024 -0.262488842 0.0288115405 -0.26200527]\n",
            "counter 600:\n",
            "[-0.000773713 0.332850784 -0.607543051 -0.86540252]\n",
            "counter 700:\n",
            "[0.106311761 -0.911064506 -0.647280395 -0.52102679]\n",
            "counter 800:\n",
            "[-0.0636531785 0.73480016 0.792608202 0.642474174]\n",
            "counter 900:\n",
            "[-0.131710023 0.726723969 -0.0839161202 -0.452219605]\n",
            "counter 1000:\n",
            "[0.0519524664 0.741318643 0.253590822 -0.869628966]\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX4AAAEICAYAAABYoZ8gAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3XuYFdWZ7/HvT4I2CoICySgNQuJd\nbmpDjBARkiAJo5hRT0AzwZnEjo4ejZPMBJ1zIqOZOUz0GR1MMspEjsmJokbHiBHHy+OFiBJpPHgD\nL6itdOuRBgUkgLHhPX/s6rZod9O7u3f37u76fZ5nP1TVqlr11kbfXaxatZYiAjMzy469Sh2AmZl1\nLid+M7OMceI3M8sYJ34zs4xx4jczyxgnfjOzjHHity5N0lZJn+3gczwm6TvJ8jmSHixi3S9KOjlZ\nnivp10Ws+3JJvyhWfZYdnyp1AGZ7EhF9O/l8twC3tLSfpJuBmoj4Hy3Ud0wx4kp+PH4dEeWpuv+5\nGHVb9viO36wDSPJNlXVZTvzW4SRVS/qBpOckbZZ0u6SyVPl5ktZKek/SYkkHp8pC0qHJ8tckrZb0\ngaRaST9I7ffnklZJ2iTpSUmj9xDPVyS9lMTyU0CpsnMlPZEsS9K1ktZL2iLpeUkjJVUC5wB/nzRF\n3Zu6zh9Keg74o6RPJdu+nDp9WXL9H0h6RtKYfNearN8s6ceS9gPuBw5OzrdV0sFNm44knZY0LW1K\nmq+OKvTvwLLFid86y38DpgEjgNHAuQCSpgD/Kyk/CHgTuK2ZOm4CvhsR/YCRwCNJHccCC4HvAgOB\nG4HFkvZpWoGkQcB/Av8DGAS8Bkxo5nxTgZOAw4H+SYwbI2IBueagn0RE34g4NXXMLGA6MCAi6vPU\nOQP4DXAgcCvwW0m9mzk/ABHxR+CrwNvJ+fpGxNtNrutwYBHwPWAwsAS4V9Leqd3y/h1Y9jjxW2eZ\nHxFvR8R7wL3A2GT7OcDCiHgmIj4ELgO+IGl4njo+Ao6WtH9EvB8RzyTbK4EbI+IPEbEzIn4JfAic\nkKeOrwEvRsSdEfERcB3w/5qJ+SOgH3AkoIhYExHvFHCd6yJiezPlK1Pn/legrJk4W+sbwH0R8VBS\n9zVAH+DEJrHl+zuwjHHit86STq7bgIaHtgeTu8sHICK2AhuBIXnqOINc4n5T0uOSvpBsPwT4ftLE\nsUnSJmBoUndTBwPrUueL9HpaRDwC/BT4GbBe0gJJ+7dwnXnrylceEbuAmmbibK2m3+Ou5Fzp77G5\nvwPLGCd+K7W3ySVuAJL27IFAbdMdI2JFRMwAPg38FrgjKVoH/FNEDEh99o2IRXnO9w65H4WG8ym9\nnuec8yPieOBock0+f9dQ1NwhzdWVSJ97L6Cc3HcAuWS8b2rfP2tFvU2/x4br+sT3aObEb6W2CPgr\nSWOTNvl/Bv4QEdXpnSTtnfSx7580ZWwBdiXF/wGcL+nzyQPZ/SRNl9Qvz/nuA46R9BdJz5uL2T3B\nps85LqmzN/BHYEfqnO8CbXm/4PjUub9HrklqeVK2CjhbUi9J04BJqePeBQZK6t9MvXcA0yV9KYn3\n+0ndT7YhRuvhnPitpCLiYeB/AneRuxv/HDCzmd3/EqiWtAU4n9zzASKiCjiPXLPM+8BamnlwGREb\ngLOAeeSalA4DljVzvv3J/ai8T64ZZSNwdVJ2E7nnDZsk/bawqwXgHnLt8e8n1/MXyQ8ZwCXAqcCm\n5Noa642Il8j9SL6enHO35qGIeBn4JnA9sCGp59SI+FMrYrOMkCdiMTPLFt/xm5lljBO/mVnGOPGb\nmWWME7+ZWcZ0yYGkBg0aFMOHDy91GGZm3cbKlSs3RMTgQvbtkol/+PDhVFVVlToMM7NuQ9KbLe+V\n46YeM7OMceI3M8sYJ34zs4zpkm38ZtZzfPTRR9TU1LBjx45Sh9IjlJWVUV5eTu/ee5zGYY+c+M2s\nQ9XU1NCvXz+GDx9ObtBQa6uIYOPGjdTU1DBixIg21+OmHjPrUDt27GDgwIFO+kUgiYEDB7b7X08t\nJn5JQyU9msx1+qKkS/LsI0nzlZs39TlJx6XKZkt6NfnMble0ZtYtOekXTzG+y0KaeuqB70fEM8n4\n5islPRQRq1P7fJXc8LaHAZ8H/h34vKQDgSuACnITSayUtDgi3m935GZm1iYtJv5kjtF3kuUPJK0h\nN51bOvHPAH6VTGO3XNIASQcBJwMPJXN8IukhcpM955sZycwyYMK8R6jd1NyUxK03ZEAfls2ZUrT6\nOtp1111HZWUl++67b8s7d5BWPdxNJsA+FvhDk6Ih7D7XaE2yrbnt+equJDdpNsOGDWtNWGadJp20\nulvC6SpqN22net70otU3fM59RaurGCKCiGCvvfK3pF933XV885vfbFXi37lzJ7169SpWiIU/3JXU\nl9wsSd+LiC1FiyAREQsioiIiKgYPLmi4CbNO15C0qudNL+pdq3W8q666iiOOOIKJEycya9Ysrrnm\nGl577TWmTZvG8ccfzxe/+EVeeuklAM4991wuvvhiTjzxRD772c9y5513NtZz9dVXM27cOEaPHs0V\nV1wBQHV1NUcccQTf+ta3GDlyJOvWreOCCy6goqKCY445pnG/+fPn8/bbbzN58mQmT54MwKJFixg1\nahQjR47khz/8YeN5+vbty/e//33GjBnDU089Vdwvo+HXaU8foDfwAPC3zZTfCMxKrb8MHATMAm5s\nbr/mPscff3yYdUWH/PB3eZeteatXr95tvdjfWyH1Pf300zFmzJjYvn17bNmyJQ499NC4+uqrY8qU\nKfHKK69ERMTy5ctj8uTJERExe/bsOPPMM2Pnzp3x4osvxuc+97mIiHjggQfivPPOi127dsXOnTtj\n+vTp8fjjj8cbb7wRkuKpp55qPOfGjRsjIqK+vj4mTZoUzz77bC7eQw6Jurq6iIiora2NoUOHxvr1\n6+Ojjz6KyZMnx9133x0REUDcfvvtea+n6Xea7F8VBeTziGi5qUe5R8g3AWsi4l+b2W0xcJGk28g9\n3N0cEe9IegD4Z0kHJPtNBS5r/c+TmVnbLVu2jBkzZlBWVkZZWRmnnnoqO3bs4Mknn+Sss85q3O/D\nDz9sXD799NPZa6+9OProo3n33XcBePDBB3nwwQc59thjAdi6dSuvvvoqw4YN45BDDuGEE05oPP6O\nO+5gwYIF1NfX884777B69WpGjx69W1wrVqzg5JNPpqGV45xzzmHp0qWcfvrp9OrVizPOOKNDvo9C\n2vgnkJsU+nlJq5JtlwPDACLiBmAJ8DVyk1xvA/4qKXtP0lXAiuS4KyN50GtmVkq7du1iwIABrFq1\nKm/5Pvvs07gcydzkEcFll13Gd7/73d32ra6uZr/99mtcf+ONN7jmmmtYsWIFBxxwAOeee26r+96X\nlZUVtV0/rcU2/oh4IiIUEaMjYmzyWRIRNyRJn+RfGhdGxOciYlREVKWOXxgRhyaf/90hV2FmtgcT\nJkzg3nvvZceOHWzdupXf/e537LvvvowYMYLf/OY3QC6pP/vss3us55RTTmHhwoVs3boVgNraWtav\nX/+J/bZs2cJ+++1H//79effdd7n//vsby/r168cHH3wAwPjx43n88cfZsGEDO3fuZNGiRUyaNKlY\nl90sD9lgZp1qyIA+Re2JM2RAnxb3GTduHKeddhqjR4/mM5/5DKNGjaJ///7ccsstXHDBBfz4xz/m\no48+YubMmYwZM6bZeqZOncqaNWv4whe+AOQewP7617/+xJ35mDFjOPbYYznyyCMZOnQoEyZMaCyr\nrKxk2rRpHHzwwTz66KPMmzePyZMnExFMnz6dGTNmtPGbKJwa/gnTlVRUVIQnYrGuaPic+xq7IqaX\nrXlr1qzhqKOOKnUYbN26lb59+7Jt2zZOOukkFixYwHHHHdfygV1Qvu9U0sqIqCjkeN/xm7Wgad99\n654qKytZvXo1O3bsYPbs2d026ReDE79ZC4r9wpGVxq233lrqELoMj85pZpYxTvxmZhnjxG9mljFO\n/GZmGeOHu2bWua4dBZvfKl59/YfBpc83W7xp0yZuvfVW/uZv/qZ458zjscceY++99+bEE0/s0PMU\ngxO/mXWuzW/B3M3Fq29u/z0Wb9q0iZ///OcFJ/6GgcyaG1a5OY899hh9+/btFonfTT1m1qPNmTOH\n1157jbFjx3LppZfypS99ieOOO45Ro0Zxzz33APmHVb7ppps4/PDDGT9+POeddx4XXXQRAHV1dZxx\nxhmMGzeOcePGsWzZMqqrq7nhhhu49tprGTt2LL///e9Leckt8h2/mfVo8+bN44UXXmDVqlXU19ez\nbds29t9/fzZs2MAJJ5zAaaedBsCrr77KL3/5S0444QTefvttrrrqKp555hn69evHlClTGodyuOSS\nS7j00kuZOHEib731Fqeccgpr1qzh/PPPp2/fvvzgBz8o5eUWxInfLA+/rdszRQSXX345S5cuZa+9\n9qK2trZxyOX0sMpPP/00kyZN4sADDwTgrLPO4pVXXgHg4YcfZvXqj2ee3bJlS+Ogbd2FE79ZHn5b\nt2e65ZZbqKurY+XKlfTu3Zvhw4c3DpecHlZ5T3bt2sXy5cspKyvryFA7lNv4zaxHSw+DvHnzZj79\n6U/Tu3dvHn30Ud588828x4wbN47HH3+c999/n/r6eu66667GsqlTp3L99dc3rjeM558+T1fnO34z\n61z9h7XYE6fV9e3BwIEDmTBhAiNHjmTcuHG89NJLjBo1ioqKCo488si8xwwZMoTLL7+c8ePHc+CB\nB3LkkUfSv38u5vnz53PhhRcyevRo6uvrOemkk7jhhhs49dRTOfPMM7nnnnu4/vrr+eIXv1i8ayyy\nQqZeXAj8ObA+IkbmKf874JxUfUcBg5PZt6qBD4CdQH2hQ4aaWQ+2hz73HaWQAdpeeOGF3dbPPvts\nKisrqa+v5+tf/zqnn346AIMGDeL222//xPGHH344zz33XHEC7mCF3PHfDPwU+FW+woi4GrgaQNKp\nwKVNplecHBEb2hmnWdeQevnoiX0GAX4O0FPNnTuXhx9+mB07djB16tTGxN8TtJj4I2KppOEF1jcL\nWNSegMy6tNTLR+XFbK6wLueaa64pdQgdpmgPdyXtC0wD7kptDuBBSSslVRbrXGbWvXTFmf66q2J8\nl8Xs1XMqsKxJM8/EiDgO+CpwoaSTmjtYUqWkKklVdXV1RQzLzEqprKyMjRs3OvkXQUSwcePGdncl\nLWavnpk0aeaJiNrkz/WS7gbGA0vzHRwRC4AFkJtzt4hxmXWYhknDhwzow7I5U0ocTddUXl5OTU0N\nvqErjrKyMsrLy9tVR1ESv6T+wCTgm6lt+wF7RcQHyfJU4MpinM+sq0hPvG759e7dmxEjRpQ6DEsp\npDvnIuBkYJCkGuAKoDdARNyQ7PZ14MGI+GPq0M8Ad0tqOM+tEfFfxQvdrJOkhxFuoc+4WXdQSK+e\nWQXsczO5bp/pba8DY9oamFmXUexhhM1KzEM2mJlljBO/mVnGOPGbmWWME7+ZWcY48ZuZZYwTv5lZ\nxjjxm5lljBO/mVnGOPGbmWWMp140a6vUFIKelMW6Eyd+s7ZKTSHoSVmsO3HiN8vjiX0uhrln51Y8\nMJv1ME78ZnmUa4MHZrMeyw93zcwyxnf8ZokJ8x6hdtN2AKrbN7OdWZfmxG+WqN20vXFGLeaWNBSz\nDuWmHjOzjGkx8UtaKGm9pBeaKT9Z0mZJq5LPj1Jl0yS9LGmtpDnFDNzMzNqmkDv+m4FpLezz+4gY\nm3yuBJDUC/gZ8FXgaGCWpKPbE6yZmbVfi4k/IpYC77Wh7vHA2oh4PSL+BNwGzGhDPWZmVkTFauP/\ngqRnJd0v6Zhk2xBgXWqfmmRbXpIqJVVJqqqrqytSWGZm1lQxEv8zwCERMQa4HvhtWyqJiAURURER\nFYMHDy5CWGZmlk+7E39EbImIrcnyEqC3pEFALTA0tWt5ss3MzEqo3Ylf0p9JUrI8PqlzI7ACOEzS\nCEl7AzOBxe09n5mZtU+LL3BJWgScDAySVANcAfQGiIgbgDOBCyTVA9uBmRERQL2ki4AHgF7Awoh4\nsUOuwszMCtZi4o+IWS2U/xT4aTNlS4AlbQvNzMw6gt/cNTPLGI/VY1Ykw+fcB8CQAX1YNmdKiaMx\na54Tv1mRNAzw1vADYNZVOfGbFYPn37VuxInfrBg8/651I364a2aWMU78ZmYZ48RvZpYxTvxmZhnj\nxG9mljHu1WOWeGKfi2Hu2bmV/sNKG4xZB3LiN0uUawPM3VzqMMw6nJt6zMwyxonfzCxjnPjNzDLG\nid/MLGOc+M3MMqbFxC9poaT1kl5opvwcSc9Jel7Sk5LGpMqqk+2rJFUVM3AzM2ubQu74bwam7aH8\nDWBSRIwCrgIWNCmfHBFjI6KibSGamVkxFTLn7lJJw/dQ/mRqdTlQ3v6wzMysoxS7jf/bwP2p9QAe\nlLRSUuWeDpRUKalKUlVdXV2RwzIzswZFe3NX0mRyiX9iavPEiKiV9GngIUkvRcTSfMdHxAKSZqKK\nioooVlxmZra7otzxSxoN/AKYEREbG7ZHRG3y53rgbmB8Mc5nZmZt1+7EL2kY8J/AX0bEK6nt+0nq\n17AMTAXy9gwyM7PO02JTj6RFwMnAIEk1wBVAb4CIuAH4ETAQ+LkkgPqkB89ngLuTbZ8Cbo2I/+qA\nazAzs1YopFfPrBbKvwN8J8/214ExnzzCrGeriUEfT7jef9huE7GbdQV+c9esyCZ+OD83vPPczbD5\nrVKHY/YJTvxmZhnjxG9mljGegcsybcK8R6jdtB2A6rISB2PWSZz4LdNqN22net703MrckoZi1mnc\n1GNmljFO/GZmGeOmHsu0J/a5GOaenVvpP6y0wZh1Eid+y7Rybcj1tzfLEDf1mJlljBO/mVnGuKnH\nrMiGDOjD8Dn3AX43wLomJ36zIls2Z8rHK3NLFoZZs9zUY2aWMU78ZmYZ48RvZpYxTvxmZhlTUOKX\ntFDSekl558xVznxJayU9J+m4VNlsSa8mn9nFCtzMzNqm0Dv+m4Fpeyj/KnBY8qkE/h1A0oHk5uj9\nPDAeuELSAW0N1szM2q+gxB8RS4H39rDLDOBXkbMcGCDpIOAU4KGIeC8i3gceYs8/IGZm1sGK1cY/\nBFiXWq9JtjW3/RMkVUqqklRVV1dXpLDMzKypLvNwNyIWRERFRFQMHjy41OGYmfVYxUr8tcDQ1Hp5\nsq257WZmViLFSvyLgW8lvXtOADZHxDvAA8BUSQckD3WnJtvMzKxEChqrR9Ii4GRgkKQacj11egNE\nxA3AEuBrwFpgG/BXSdl7kq4CViRVXRkRe3pIbGZmHaygxB8Rs1ooD+DCZsoWAgtbH5qZmXWELvNw\n18zMOocTv5lZxjjxm5lljBO/mVnGOPGbmWWME7+ZWcY48ZuZZYwTv5lZxjjxm5lljBO/mVnGOPGb\nmWVMQWP1mFnb1MQgyuf2z630HwaXPl/agMzwHb9Zh5r44XyYuzn32fxWqcMxA5z4zcwyx009lj3X\njmq8+66JQZSXOByzzubEb9mz+S2G77gVgCED+rCsxOGYdTYnfsuk6nnTSx2CWckU1MYvaZqklyWt\nlTQnT/m1klYln1ckbUqV7UyVLS5m8GZm1not3vFL6gX8DPgKUAOskLQ4IlY37BMRl6b2/+/Asakq\ntkfE2OKFbGZm7VHIHf94YG1EvB4RfwJuA2bsYf9ZwKJiBGdmZsVXSOIfAqxLrdck2z5B0iHACOCR\n1OYySVWSlks6vbmTSKpM9quqq6srICwzM2uLYvfjnwncGRE7U9sOiYgK4GzgOkmfy3dgRCyIiIqI\nqBg8eHCRwzIzswaFJP5aYGhqvTzZls9MmjTzRERt8ufrwGPs3v5vZmadrJDunCuAwySNIJfwZ5K7\ne9+NpCOBA4CnUtsOALZFxIeSBgETgJ8UI3Cz7mDIgD4Mn3MfANVlJQ7GLNFi4o+IekkXAQ8AvYCF\nEfGipCuBqoho6KI5E7gtIiJ1+FHAjZJ2kfvXxbx0byCznm7ZnCkfr8wtWRhmuynoBa6IWAIsabLt\nR03W5+Y57klgVDviMzOzIvMgbWZmGePEb2aWMU78ZmYZ48RvZpYxTvxmZhnjYZktGzz5ilkjJ37L\nhs1v5ea9BSbOuY/q0kZjVlJu6jEzyxgnfjOzjHHiNzPLGCd+M7OMceI3M8sYJ34zs4xx4jczyxgn\nfjOzjPELXGadpCYGUT63f26l/zC49PnSBmSZ5Tt+s04y8cP5ubeH525uHD7CrBQKSvySpkl6WdJa\nSXPylJ8rqU7SquTznVTZbEmvJp/ZxQzezMxar8WmHkm9gJ8BXwFqgBWSFueZO/f2iLioybEHAlcA\nFUAAK5Nj3y9K9Gat0DDp+ZABfUociVlpFdLGPx5YGxGvA0i6DZgBFDJp+inAQxHxXnLsQ8A0YFHb\nwjVru+p500sdglmXUEhTzxBgXWq9JtnW1BmSnpN0p6ShrTwWSZWSqiRV1dXVFRCWmZm1RbEe7t4L\nDI+I0cBDwC9bW0FELIiIioioGDx4cJHCMus6hgzow/A59zU2OZmVSiFNPbXA0NR6ebKtUURsTK3+\nAvhJ6tiTmxz7WGuDNOsJls2Z8vHK3JKFYVbQHf8K4DBJIyTtDcwEFqd3kHRQavU0YE2y/AAwVdIB\nkg4ApibbzMysRFq844+IekkXkUvYvYCFEfGipCuBqohYDFws6TSgHngPODc59j1JV5H78QC4suFB\nr5mZlUZBb+5GxBJgSZNtP0otXwZc1syxC4GF7YjRzMyKyG/umplljMfqMSsBj9tjpeQ7frMS8Lg9\nVkq+47ee69pRjUm1JgZRXuJwzLoKJ37ruTa/lbujBibOuY/q0kZj1mW4qcfMLGOc+M3MMsaJ38ws\nY5z4zcwyxg93rUfz5Ctmn+TEbz2aJ18x+yQ39ZiZZYwTv5lZxjjxm5lljBO/mVnGOPGbmWWMe/WY\nlUDDxOsA1WUlDsYyp6DEL2ka8G/kpl78RUTMa1L+t8B3yE29WAf8dUS8mZTtBBoGG38rIk4rUuxm\n3ZYnXrdSajHxS+oF/Az4ClADrJC0OCJWp3b7v0BFRGyTdAHwE+AbSdn2iBhb5LjNegxPymKdrZA2\n/vHA2oh4PSL+BNwGzEjvEBGPRsS2ZHU5eOhzs0J5UhbrbIU09QwB1qXWa4DP72H/bwP3p9bLJFWR\nawaaFxG/zXeQpEqgEmDYsGEFhGWWhydfMWtRUR/uSvomUAFMSm0+JCJqJX0WeETS8xHxWtNjI2IB\nsACgoqIiihmXZYgnXzFrUSFNPbXA0NR6ebJtN5K+DPwDcFpEfNiwPSJqkz9fBx4Djm1HvGZm1k6F\nJP4VwGGSRkjaG5gJLE7vIOlY4EZySX99avsBkvZJlgcBE4D0Q2EzM+tkLTb1RES9pIuAB8h151wY\nES9KuhKoiojFwNVAX+A3kuDjbptHATdK2kXuR2Zek95AZmbWyQpq44+IJcCSJtt+lFr+cjPHPQmM\nak+AZq3lMfjN9sxv7lqP4zH4zfbMY/WYmWWM7/it+3PffbNWceK37q8n9d3vPww8fIN1MCd+s64k\nnegbfgDMisyJ36zE0kM0DxnQZ/eRO806gBO/WYmlE33DD4BZR3KvHjOzjPEdv/UIfmnLrHBO/NYj\n9JSXtjwlo3UGJ36zLiTd3l9zhWfmso7hNn6zLsozc1lH8R2/dU9+W9eszZz4rXvqSW/rNsPt/dZR\nnPite0jd4QO5Nu8ebrcXueaWLAzrgZz4rXtI3eEDTJj3CLXuwmnWJk781i3VbtreY7pwmnW2ghK/\npGnAv5GbevEXETGvSfk+wK+A44GNwDciojopuwz4NrATuDgiHiha9JYp6eEMsnaXXxPu2mnF02Li\nl9QL+BnwFaAGWCFpcZO5c78NvB8Rh0qaCfwL8A1JR5ObnP0Y4GDgYUmHR8TOYl+I9TzvzD2Ug6jL\nLTM403f43+jzH9Ru2g7AU1zCQc2N3OkfBStAIXf844G1EfE6gKTbgBlAOvHP4OPHT3cCP1Vu1vUZ\nwG0R8SHwhqS1SX1PFSd86xGaPrhN7IxB8I+5dv2DOjumLib9oHfCvD6NPwJN7fajkP4RSH/H/nHI\nvEIS/xBgXWq9Bvh8c/tERL2kzcDAZPvyJscOyXcSSZVAZbK6VdLLBcSWzyBgQxuP7Wp6yrW08Tq2\nwJUqejDt1KX/Tg7ebe0F+Nt831/j9i59La3QU64D2ncthxS6Y5d5uBsRC4AF7a1HUlVEVBQhpJLr\nKdfSU64DfC1dUU+5Dui8aylkyIZaYGhqvTzZlncfSZ8C+pN7yFvIsWZm1okKSfwrgMMkjZC0N7mH\ntYub7LMYmJ0snwk8EhGRbJ8paR9JI4DDgKeLE7qZmbVFi009SZv9RcAD5LpzLoyIFyVdCVRFxGLg\nJuD/JA9v3yP340Cy3x3kHgTXAxd2Qo+edjcXdSE95Vp6ynWAr6Ur6inXAZ10LcrdmJuZWVZ4WGYz\ns4xx4jczy5gemfglXSXpOUmrJD0o6eCWj+p6JF0t6aXkWu6WNKDUMbWVpLMkvShpl6Ru1/VO0jRJ\nL0taK2lOqeNpD0kLJa2X9EKpY2kPSUMlPSppdfLf1iWljqmtJJVJelrSs8m1/GOHnq8ntvFL2j8i\ntiTLFwNHR8T5JQ6r1SRNJddDql7SvwBExA9LHFabSDoK2AXcCPwgIqpKHFLBkmFLXiE1bAkwq8mw\nJd2GpJOArcCvImJkqeNpK0kHAQdFxDOS+gErgdO7499LMtLBfhGxVVJv4AngkohY3sKhbdIj7/gb\nkn5iP6Bb/rpFxIMRUZ+sLofuO9FURKyJiLa+jV1qjcOWRMSfgIZhS7qliFhKrvddtxYR70TEM8ny\nB8AamhkZoKuLnK3Jau/k02F5q0cmfgBJ/yRpHXAO8KNSx1MEfw3cX+ogMirfsCXdMsH0VJKGA8cC\nfyhtJG0nqZekVcB64KGI6LBr6baJX9LDkl7I85kBEBH/EBFDgVuAi0obbfNauo5kn38g9x7ELaWL\ntGWFXItZsUnqC9wFfK/Jv/a7lYjYGRFjyf3LfrykDmuG6zJj9bRWRHy5wF1vAZYAV3RgOG3W0nVI\nOhf4c+BL0cUfyLTi76S78dAxQ2bLAAABBElEQVQjXVTSHn4XcEtE/Gep4ymGiNgk6VFgGtAhD+C7\n7R3/nkg6LLU6A3ipVLG0RzIBzt8Dp0XEtlLHk2GFDFtinSx5IHoTsCYi/rXU8bSHpMENvfYk9SHX\nkaDD8lZP7dVzF3AEuV4kbwLnR0S3u0NLhsDYh9yAdwDLu2PvJABJXweuBwYDm4BVEXFKaaMqnKSv\nAdfx8bAl/1TikNpM0iLgZHJDAL8LXBERN5U0qDaQNBH4PfA8uf/XAS6PiCWli6ptJI0Gfknuv6+9\ngDsi4soOO19PTPxmZta8HtnUY2ZmzXPiNzPLGCd+M7OMceI3M8sYJ34zs4xx4jczyxgnfjOzjPn/\nvqkUPGYXj9EAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "(1000, 4)\n",
            "[-0.324534267 0.440980792 -0.0676107109 -0.188951045]\n",
            "-0.00844934583 -0.500320792\n",
            "Time for epoch 1000,\n",
            "counter 1100:\n",
            "[-0.171408787 0.566211402 0.424108744 0.507902324]\n",
            "counter 1200:\n",
            "[-0.420846432 -2.19419885 -1.64170992 -0.608803809]\n",
            "counter 1300:\n",
            "[-0.108104184 -1.09888387 -0.0870935097 -0.109131068]\n",
            "counter 1400:\n",
            "[0.308057219 0.0352689028 -0.211195141 0.259908468]\n",
            "counter 1500:\n",
            "[0.4316926 -0.203209311 0.0708409399 0.131833434]\n",
            "counter 1600:\n",
            "[-0.881487 -1.36375332 -0.541855156 0.0246205404]\n",
            "counter 1700:\n",
            "[0.729678 0.557494044 -0.713561058 0.637786]\n",
            "counter 1800:\n",
            "[-0.494118482 -0.470868945 -1.03106177 0.0263707638]\n",
            "counter 1900:\n",
            "[-0.733730435 0.512865603 0.437443852 0.552434266]\n",
            "counter 2000:\n",
            "[-0.126240328 -1.15631282 1.0068537 0.91185838]\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX4AAAEICAYAAABYoZ8gAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3Xt4VfWd7/H3R4pEAYMS7CABQutd\nbmqgKihCKzKlih31FLVTndOaauvROu2MaOdojrYzzOgzerD2KKfyaE8VtVorKo6X8VZRKuhBRfCC\nGiWRIxEFSgVq4Hv+2CtxE3PZSXayk6zP63n2w17370r0s1d+67d/SxGBmZmlx26FLsDMzLqWg9/M\nLGUc/GZmKePgNzNLGQe/mVnKOPjNzFLGwW/dmqQtkr7Uycd4UtL3kvdnSXokj/t+VdLxyftKSb/J\n474vk/SrfO3P0uMLhS7ArCURMaCLj3cbcFtr60m6BaiOiH9qZX+H5aOu5MPjNxFRmrXvf87Hvi19\nfMVv1gkk+aLKui0Hv3U6SVWSfiLpZUmbJN0pqShr+bmS1kj6SNIiSftlLQtJ+yfvvy5plaQ/SaqR\n9JOs9b4haYWkjZKelTS2hXpOkPRaUssvAGUtO0fSM8l7SbpW0npJmyW9Imm0pArgLOAfk6ao+7PO\n8xJJLwN/lvSFZN7Xsg5flJz/nyS9KGlcU+eaTN8i6WeS+gMPAfslx9siab/GTUeSTk6aljYmzVeH\n5Po7sHRx8FtX+S/ADGAUMBY4B0DSNOBfkuVDgXeBO5rZx83A9yNiIDAaeDzZx+HAAuD7wGDgJmCR\npH6NdyCpBPgd8E9ACfAWMKmZ400HjgMOBIqTGjdExHwyzUH/FhEDIuKkrG3OAGYCgyKirol9zgJ+\nC+wD3A78XlLfZo4PQET8Gfhr4P3keAMi4v1G53UgsBD4ETAEWAzcL2n3rNWa/B1Y+jj4ravMi4j3\nI+Ij4H5gfDL/LGBBRLwYEduBS4GjJZU1sY9PgUMl7RURH0fEi8n8CuCmiPhjROyIiFuB7cBRTezj\n68CrEXF3RHwKXAf8v2Zq/hQYCBwMKCJWR8S6HM5zbURsbWb5C1nH/negqJk62+pbwIMR8Wiy72uA\nPYBjGtXW1O/AUsbBb10lO1w/Aepv2u5H5iofgIjYAmwAhjWxj1PJBPe7kp6SdHQyfyTw46SJY6Ok\njcDwZN+N7QeszTpeZE9ni4jHgV8ANwDrJc2XtFcr59nkvppaHhE7gepm6myrxj/Hncmxsn+Ozf0O\nLGUc/FZo75MJbgCS9uzBQE3jFSNiWUTMAvYFfg/clSxaC/w8IgZlvfaMiIVNHG8dmQ+F+uMpe7qJ\nY86LiCOBQ8k0+fxD/aLmNmluX4nsY+8GlJL5GUAmjPfMWvev2rDfxj/H+vP63M/RzMFvhbYQ+DtJ\n45M2+X8G/hgRVdkrSdo96WNfnDRlbAZ2Jov/N3CepK8kN2T7S5opaWATx3sQOEzS3yQ9by5k14DN\nPuaEZJ99gT8D27KO+QHQnu8XHJl17B+RaZJamixbAZwpqY+kGcCUrO0+AAZLKm5mv3cBMyV9Nan3\nx8m+n21HjdbLOfitoCLiMeC/A/eQuRr/MjC7mdX/FqiStBk4j8z9ASJiOXAumWaZj4E1NHPjMiI+\nBE4H5pJpUjoAWNLM8fYi86HyMZlmlA3A1cmym8ncb9go6fe5nS0A95Fpj/84OZ+/ST7IAC4CTgI2\nJufWsN+IeI3Mh+TbyTF3aR6KiNeBbwPXAx8m+zkpIv7ShtosJeQHsZiZpYuv+M3MUsbBb2aWMg5+\nM7OUcfCbmaVMtxxIqqSkJMrKygpdhplZj/HCCy98GBFDclm3WwZ/WVkZy5cvL3QZZmY9hqR3W18r\nw009ZmYp4+A3M0sZB7+ZWcp0yzZ+M+s9Pv30U6qrq9m2bVuhS+kVioqKKC0tpW/fFh/j0CIHv5l1\nqurqagYOHEhZWRmZQUOtvSKCDRs2UF1dzahRo9q9n1abeiQtSB49t7KZ5f+QPPJuhaSVknZI2idZ\nVpU8rm6FJHfTMUuhbdu2MXjwYId+Hkhi8ODBHf7rKZc2/lvIPK6tSRFxdUSMj4jxZJ6e9FTyhJ96\nU5Pl5R2q1Mx6LId+/uTjZ9lq8EfE08BHra2XOIPM0LFmZtZN5a2NX9KeZP4yuCBrdgCPSAoyz0Sd\n38L2FWSencqIESPyVZaZdTOT5j5OzcbmHkncdsMG7cGSOdPytr/Odt1111FRUcGee+7Z+sqdJJ83\nd08CljRq5pkcETWS9gUelfRa8hfE5yQfCvMBysvL/ZAA6z6uHQOb3su8Lx4BF79S2Hp6uJqNW6ma\nOzNv+yub82De9pUPEUFEsNtuTTeoXHfddXz7299uU/Dv2LGDPn365KvEvPbjn02jZp6IqEn+XQ/c\nC0zM4/HMusam96ByU+YFUFmceV07prB1WZtcddVVHHTQQUyePJkzzjiDa665hrfeeosZM2Zw5JFH\ncuyxx/Laa68BcM4553DhhRdyzDHH8KUvfYm77767YT9XX301EyZMYOzYsVxxxRUAVFVVcdBBB/Gd\n73yH0aNHs3btWs4//3zKy8s57LDDGtabN28e77//PlOnTmXq1KkALFy4kDFjxjB69GguueSShuMM\nGDCAH//4x4wbN47nnnsuvz+M+k+nll5AGbCyheXFZO4D9M+a1x8YmPX+WWBGLsc78sgjw6zbuGKv\nGHnJAzHykgfimH/5z13mW+tWrVq1y/TISx7I6/5z2d/zzz8f48aNi61bt8bmzZtj//33j6uvvjqm\nTZsWb7zxRkRELF26NKZOnRoREWeffXacdtppsWPHjnj11Vfjy1/+ckREPPzww3HuuefGzp07Y8eO\nHTFz5sx46qmn4p133glJ8dxzzzUcc8OGDRERUVdXF1OmTImXXnopU+/IkVFbWxsRETU1NTF8+PBY\nv359fPrppzF16tS49957IyICiDvvvLPJ82n8M03WXx455GtEtN7UI2khcDxQIqkauALom3xo3Jis\n9k3gkYj4c9amXwTuTe5AfwG4PSL+o92fUGYFVN800d2aFSw3S5YsYdasWRQVFVFUVMRJJ53Etm3b\nePbZZzn99NMb1tu+fXvD+1NOOYXddtuNQw89lA8++ACARx55hEceeYTDDz8cgC1btvDmm28yYsQI\nRo4cyVFHHdWw/V133cX8+fOpq6tj3bp1rFq1irFjx+5S17Jlyzj++OMZMiQzqOZZZ53F008/zSmn\nnEKfPn049dRTO+Xn0WrwR8QZOaxzC5lun9nz3gbGtbcwM7POtHPnTgYNGsSKFSuaXN6vX7+G95E8\nmzwiuPTSS/n+97+/y7pVVVX079+/Yfqdd97hmmuuYdmyZey9996cc845be57X1RUlNd2/Wweq8es\nDYYN2oOyOQ/6yr+HmTRpEvfffz/btm1jy5YtPPDAA+y5556MGjWK3/72t0Am1F966aUW93PiiSey\nYMECtmzZAkBNTQ3r16//3HqbN2+mf//+FBcX88EHH/DQQw81LBs4cCB/+tOfAJg4cSJPPfUUH374\nITt27GDhwoVMmTIlX6fdLA/ZYNYGu3QbrCxYGT1a/YdnPvfXmgkTJnDyySczduxYvvjFLzJmzBiK\ni4u57bbbOP/88/nZz37Gp59+yuzZsxk3rvmGiunTp7N69WqOPvpoIHMD9je/+c3nrszHjRvH4Ycf\nzsEHH8zw4cOZNGlSw7KKigpmzJjBfvvtxxNPPMHcuXOZOnUqEcHMmTOZNWtWO38SuVP9nzDdSXl5\nefhBLNZtVBZ/1qMnl/m2i9WrV3PIIYcUugy2bNnCgAED+OSTTzjuuOOYP38+RxxxRKHLapemfqaS\nXogcR0jwFb+ZpUJFRQWrVq1i27ZtnH322T029PPBwW9mqXD77bcXuoRuwzd3zcxSxsFvZpYyDn4z\ns5Rx8JuZpYxv7ppZ18oe7TQfWhkxdePGjdx+++384Ac/yN8xm/Dkk0+y++67c8wxx3TqcfLBwW9m\nXat+tNN8qSxucfHGjRv55S9/mXPw1w9k1tywys158sknGTBgQI8Ifjf1mFmvNmfOHN566y3Gjx/P\nxRdfzFe/+lWOOOIIxowZw3333Qc0PazyzTffzIEHHsjEiRM599xzueCCzDOmamtrOfXUU5kwYQIT\nJkxgyZIlVFVVceONN3Lttdcyfvx4/vCHPxTylFvlK34z69Xmzp3LypUrWbFiBXV1dXzyySfstdde\nfPjhhxx11FGcfPLJALz55pvceuutHHXUUbz//vtcddVVvPjiiwwcOJBp06Y1DOVw0UUXcfHFFzN5\n8mTee+89TjzxRFavXs15553HgAED+MlPflLI082Jg9/MUiMiuOyyy3j66afZbbfdqKmpaRhyOXtY\n5eeff54pU6awzz77AHD66afzxhtvAPDYY4+xatWqhn1u3ry5YdC2nsLBb2apcdttt1FbW8sLL7xA\n3759KSsraxguOXtY5Zbs3LmTpUuXUlRU1Jmldiq38Zu10zqGNDyGcV3l/oUux5qRPQzypk2b2Hff\nfenbty9PPPEE7777bpPbTJgwgaeeeoqPP/6Yuro67rnnnoZl06dP5/rrr2+Yrh/PP/s43Z2v+M3a\naWjlmqz3LfcssSzFI1rtidPm/bVg8ODBTJo0idGjRzNhwgRee+01xowZQ3l5OQcffHCT2wwbNozL\nLruMiRMnss8++3DwwQdTXJyped68efzwhz9k7Nix1NXVcdxxx3HjjTdy0kkncdppp3Hfffdx/fXX\nc+yxx+bvHPPMwW9mXauFPvedJZcB2lauXLnL9JlnnklFRQV1dXV885vf5JRTTgGgpKSEO++883Pb\nH3jggbz88sv5KbiTuanHzKwJlZWVjB8/ntGjRzNq1KiG4O8NfMVvZtaEa665ptAldBoHv1lTsoYV\nqI4SSgtcTk8XEUgqdBm9Qj6emthqU4+kBZLWS1rZzPLjJW2StCJ5XZ61bIak1yWtkTSnw9WadZX6\nYQUqNzF5+7xCV9OjFRUVsWHDhrwEVtpFBBs2bOhwV9JcrvhvAX4B/LqFdf4QEd/IniGpD3ADcAJQ\nDSyTtCgiVjW1A7Pupv6B4Lk8zNuaV1paSnV1NbW1tYUupVcoKiqitLRjf4O2GvwR8bSksnbseyKw\nJiLeBpB0BzALcPBbj1A1d2ahS+gV+vbty6hRowpdhmXJV6+eoyW9JOkhSYcl84YBa7PWqU7mNUlS\nhaTlkpb7ysDMrPPkI/hfBEZGxDjgeuD37dlJRMyPiPKIKB8yZEgeyjIzs6Z0OPgjYnNEbEneLwb6\nSioBaoDhWauWJvPMzKyAOhz8kv5KST8tSROTfW4AlgEHSBolaXdgNrCoo8czM7OOafXmrqSFwPFA\niaRq4AqgL0BE3AicBpwvqQ7YCsyOTL+tOkkXAA8DfYAFEfFqp5yFmZnlLJdePWe0svwXZLp7NrVs\nMbC4faWZmVln8Fg9ZmYp4yEbzOp5mAZLCQe/Wb36YRqAyXMepKqw1Zh1Gjf1mJmljK/4zfKgOkoo\nrX+qVPGIgjxsxCxXvuI3y4PJ2+c1jOZZf5/ArLvyFb9ZHgwbtEfDaJ5VHRsx16zTOfjN8mDJnGmf\nTVQWrAyznLipx8wsZXzFb5bFD1+xNHDwm2Xxw1csDdzUY2aWMg5+M7OUcfCbmaWMg9/MLGUc/GZm\nKePgNzNLGQe/mVnKOPjNzFLGwW9mljKtBr+kBZLWS1rZzPKzJL0s6RVJz0oal7WsKpm/QtLyfBZu\nZmbtk8sV/y3AjBaWvwNMiYgxwFXA/EbLp0bE+Igob1+JZmaWT62O1RMRT0sqa2H5s1mTS8HPqDYz\n687y3cb/XeChrOkAHpH0gqSKljaUVCFpuaTltbW1eS7LzMzq5W10TklTyQT/5KzZkyOiRtK+wKOS\nXouIp5vaPiLmkzQTlZeXR77qMjOzXeXlil/SWOBXwKyI2FA/PyJqkn/XA/cCE/NxPDMza78OB7+k\nEcDvgL+NiDey5veXNLD+PTAdaLJnkJmZdZ1Wm3okLQSOB0okVQNXAH0BIuJG4HJgMPBLSQB1SQ+e\nLwL3JvO+ANweEf/RCedgZmZtkEuvnjNaWf494HtNzH8bGPf5LczMrJD8zV0zs5Rx8JuZpYwftm7W\nCcrmPAjAsEF7sGTOtAJXY7YrB7+l27VjYNN7AFRHSd6+dl41dybw2QeAWXfi4Ld02/QeZdtuB5Kr\n8wKXY9YVHPyWevVX52Zp4Zu7ZmYp4+A3M0sZB7+ZWco4+M3MUsbBb2aWMg5+M7OUcXdOs3wrHgGV\nxQA8068EcHdR614c/Gb5dvErDW9Lkw8As+7ETT1mZinj4DczSxkHv5lZyjj4zcxSxsFvZpYyDn4z\ns5Rx8JuZpUxOwS9pgaT1klY2s1yS5klaI+llSUdkLTtb0pvJ6+x8FW5mZu2T6xX/LcCMFpb/NXBA\n8qoA/heApH2AK4CvABOBKyTt3d5izcys43IK/oh4GviohVVmAb+OjKXAIElDgROBRyPio4j4GHiU\nlj9AzMysk+WrjX8YsDZrujqZ19z8z5FUIWm5pOW1tbV5KsvMzBrrNjd3I2J+RJRHRPmQIUMKXY6Z\nWa+Vr+CvAYZnTZcm85qbb2ZmBZKv4F8EfCfp3XMUsCki1gEPA9Ml7Z3c1J2ezDMzswLJaVhmSQuB\n44ESSdVkeur0BYiIG4HFwNeBNcAnwN8lyz6SdBWwLNnVlRHR0k1iMzPrZDkFf0Sc0cryAH7YzLIF\nwIK2l2ZmZp2h29zcNTOzruHgNzNLGQe/mVnKOPjNzFLGwW9mljIOfjOzlMmpO6eZtU91lFBaWZyZ\nKB4BF79S2ILM8BW/WaeavH0eVG7KvDa9V+hyzAAHv5lZ6jj4zcxSxm38Zp1o2KA9KJvzIABVRQUu\nxizh4DfrREvmTPtsorJgZZjtwk09ZmYp4+A3M0sZN/VY+lw7pqFrZXWUUFrgcsy6moPf0mfTe5Rt\nux3I3HxdUuByzLqag99SqWruzEKXYFYwbuM3M0sZB7+ZWco4+M3MUsbBb2aWMjkFv6QZkl6XtEbS\nnCaWXytpRfJ6Q9LGrGU7spYtymfxZmbWdq326pHUB7gBOAGoBpZJWhQRq+rXiYiLs9b/b8DhWbvY\nGhHj81eymZl1RC5X/BOBNRHxdkT8BbgDmNXC+mcAC/NRnJmZ5V8uwT8MWJs1XZ3M+xxJI4FRwONZ\ns4skLZe0VNIpzR1EUkWy3vLa2tocyjIzs/bI983d2cDdEbEja97IiCgHzgSuk/TlpjaMiPkRUR4R\n5UOGDMlzWWZmVi+X4K8BhmdNlybzmjKbRs08EVGT/Ps28CS7tv+bmVkXyyX4lwEHSBolaXcy4f65\n3jmSDgb2Bp7Lmre3pH7J+xJgErCq8bZmZtZ1Wu3VExF1ki4AHgb6AAsi4lVJVwLLI6L+Q2A2cEdE\nRNbmhwA3SdpJ5kNmbnZvIDMz63o5DdIWEYuBxY3mXd5ourKJ7Z4FxnSgPjMzyzN/c9fMLGUc/GZm\nKePx+M26SHWUUFpZnJkoHgEXv1LYgiy1fMVv1kUmb58HlZsyr+TRj2aF4Ct+sy4ybNAelM15EICq\nogIXY6nm4DfrIkvmTPtsorJgZZi5qcfMLG0c/GZmKePgNzNLGQe/mVnK+OaupcKkuY9Ts3Er4B41\nZg5+S4WajVupmjszM1FZ0FLMCs7Bb6nwTL8LofLMzETxiMIWY1ZgDn5LhVJ9mPnGrJn55q6ZWdo4\n+M3MUsbBb2aWMg5+M7OUcfCbmaWMg9/MLGVyCn5JMyS9LmmNpDlNLD9HUq2kFcnre1nLzpb0ZvI6\nO5/Fm5lZ27Xaj19SH+AG4ASgGlgmaVFErGq06p0RcUGjbfcBrgDKgQBeSLb9OC/Vm5lZm+VyxT8R\nWBMRb0fEX4A7gFk57v9E4NGI+CgJ+0eBGe0r1czM8iGX4B8GrM2ark7mNXaqpJcl3S1peBu3RVKF\npOWSltfW1uZQlpmZtUe+bu7eD5RFxFgyV/W3tnUHETE/IsojonzIkCF5KsvMzBrLJfhrgOFZ06XJ\nvAYRsSEitieTvwKOzHVbMzPrWrkE/zLgAEmjJO0OzAYWZa8gaWjW5MnA6uT9w8B0SXtL2huYnswz\nM7MCabVXT0TUSbqATGD3ARZExKuSrgSWR8Qi4EJJJwN1wEfAOcm2H0m6isyHB8CVEfFRJ5yHmZnl\nKKdhmSNiMbC40bzLs95fClzazLYLgAUdqNGsXdZV7s9QMh0F1jGEoa2sb5YWHo/feq2h1DaMwe/Q\nN/uMh2wwM0sZB7+ZWco4+M3MUsbBb2aWMg5+M7OUcfCbmaWMu3OaFcA6hjC0sjjr/ZoCV2Rp4uA3\nK4DsoK//ADDrKm7qMTNLGQe/mVnKOPjNzFLGwW9mljIOfjOzlHHwm5mljIPfzCxlHPxmZinj4Dcz\nSxkHv5lZynjIButVJs19nJqNWwGoKipwMWbdlIPfepWajVupmjszM1FZ0FLMuq2cmnokzZD0uqQ1\nkuY0sfzvJa2S9LKk/5Q0MmvZDkkrkteifBZvZmZt1+oVv6Q+wA3ACUA1sEzSoohYlbXa/wXKI+IT\nSecD/wZ8K1m2NSLG57luMzNrp1yu+CcCayLi7Yj4C3AHMCt7hYh4IiI+SSaXAqX5LdPMzPIllzb+\nYcDarOlq4CstrP9d4KGs6SJJy4E6YG5E/L6pjSRVABUAI0aMyKEss96jbM6DAAwbtAdL5kwrcDXW\n2+X15q6kbwPlwJSs2SMjokbSl4DHJb0SEW813jYi5gPzAcrLyyOfdVl6PNPvQqg8MzNR3HMuIOpv\nSNd/AJh1plyCvwYYnjVdmszbhaSvAT8FpkTE9vr5EVGT/Pu2pCeBw4HPBb9ZPpTqQ6jcVOgy2qZ4\nBCRP4XqmXwkws7D1WK+XS/AvAw6QNIpM4M8GzsxeQdLhwE3AjIhYnzV/b+CTiNguqQSYRObGr5nV\nu/iVhrelfgyjdYFWgz8i6iRdADwM9AEWRMSrkq4ElkfEIuBqYADwW0kA70XEycAhwE2SdpK5kTy3\nUW8gMzPrYjm18UfEYmBxo3mXZ73/WjPbPQuM6UiBZmaWXx6rx8wsZRz8ZmYp4+A3M0sZB7+ZWco4\n+M3MUsbDMpt1I9VR8llf/uIRu/TxN8sXX/GbdSOTt8/LfPO4chNseq/Q5Vgv5St+s25k2KA9Gsbr\n8RPErLM4+K3HW1e5P0OpzbxnCEMLXE9H7DIyZ2XByrBezsFvPd5QahsGZuvJoW/WVdzGb2aWMg5+\nM7OUcfCbmaWM2/itR5o093FqNm4F3PvFrK0c/NYj1Wzc2vC4wt7a+2UdQxiafJkr835NgSuy3sLB\nbz1ST322bltkB/1QP5nL8sjBbz1Sj3y2rlk34eC3HiG7TR/S2a5f/43eYYP22PWLXmZt5OC3HmGX\nNn3ote36Lak///oPALP2cvBbj7BLmz702nb9ZhWPgKSd/5l+JcDMltc3a4GD37qt7DF4+mhIutv0\ns4ZnLvWNXuugnIJf0gzgfwJ9gF9FxNxGy/sBvwaOBDYA34qIqmTZpcB3gR3AhRHxcN6qt17NY/A0\nz+391hGtBr+kPsANwAlANbBM0qKIWJW12neBjyNif0mzgX8FviXpUGA2cBiwH/CYpAMjYke+T8R6\nB38xKwfFI6gi0+xVvbUEeKuw9ViPk8sV/0RgTUS8DSDpDmAWkB38s/jsdtvdwC8kKZl/R0RsB96R\ntCbZ33P5Kd96quyAf6bfhZnumcASgPrAT1s7fq6ymn36VO7f0PbfWHWUZB7sgv8ysF3lEvzDgLVZ\n09XAV5pbJyLqJG0CBifzlzbadlhTB5FUAVQkk1skvZ5DbU0pAT5s57bdTW85lxbPY3izm62Ev1dn\n1NMRPeh3shn4BgDvArr0cyv0oHNpUW85D+jYuYzMdcVuc3M3IuYD8zu6H0nLI6I8DyUVXG85l95y\nHuBz6Y56y3lA151LLqNz1rDrRVlpMq/JdSR9ASgmc5M3l23NzKwL5RL8y4ADJI2StDuZm7WLGq2z\nCDg7eX8a8HhERDJ/tqR+kkYBBwDP56d0MzNrj1abepI2+wuAh8l051wQEa9KuhJYHhGLgJuB/5Pc\nvP2IzIcDyXp3kbkRXAf8sAt69HS4uagb6S3n0lvOA3wu3VFvOQ/oonNR5sLczMzSwk/gMjNLGQe/\nmVnK9Mrgl3SVpJclrZD0iKT9Cl1Te0i6WtJrybncK2lQoWtqL0mnS3pV0k5JPa7rnaQZkl6XtEbS\nnELX0xGSFkhaL2lloWvpCEnDJT0haVXy39ZFha6pvSQVSXpe0kvJufyPTj1eb2zjl7RXRGxO3l8I\nHBoR5xW4rDaTNJ1MD6k6Sf8KEBGXFLisdpF0CLATuAn4SUQsL3BJOUuGLXmDrGFLgDMaDVvSY0g6\nDtgC/DoiRhe6nvaSNBQYGhEvShoIvACc0hN/L8lIB/0jYoukvsAzwEURsbSVTdulV17x14d+oj/Q\nIz/dIuKRiKhLJpeS+R5EjxQRqyOivd/GLrSGYUsi4i9A/bAlPVJEPE2m912PFhHrIuLF5P2fgNU0\nMzJAdxcZW5LJvsmr03KrVwY/gKSfS1oLnAVcXuh68uC/Ag8VuoiUamrYkh4ZML2VpDLgcOCPha2k\n/ST1kbQCWA88GhGddi49NvglPSZpZROvWQAR8dOIGA7cBlxQ2Gqb19p5JOv8lMz3IG4rXKWty+Vc\nzPJN0gDgHuBHjf7a71EiYkdEjCfzl/1ESZ3WDNdtxuppq4j4Wo6r3gYsBq7oxHLarbXzkHQOmZG2\nvhrd/IZMG34nPY2HHummkvbwe4DbIuJ3ha4nHyJio6QngBlAp9yA77FX/C2RdEDW5CzgtULV0hHJ\nA3D+ETg5Ij4pdD0plsuwJdbFkhuiNwOrI+LfC11PR0gaUt9rT9IeZDoSdFpu9dZePfcAB5HpRfIu\ncF5E9LgrtGQIjH5kBrwDWNoTeycBSPomcD0wBNgIrIiIEwtbVe4kfR24js+GLfl5gUtqN0kLgePJ\nDAH8AXBFRNxc0KLaQdJk4A/AK2T+Xwe4LCIWF66q9pE0FriVzH9fuwF3RcSVnXa83hj8ZmbWvF7Z\n1GNmZs1z8JuZpYyD38wsZRzySDERAAAAG0lEQVT8ZmYp4+A3M0sZB7+ZWco4+M3MUub/Azq+Nxmi\nxluaAAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "(1000, 4)\n",
            "[-0.139301375 -0.982219338 0.808072329 1.00963449]\n",
            "-0.00844934583 -0.500320792\n",
            "Time for epoch 2000,\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/ops/resource_variable_ops.py:1781: calling BaseResourceVariable.__init__ (from tensorflow.python.ops.resource_variable_ops) with constraint is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "If using Keras pass *_constraint arguments to layers.\n",
            "INFO:tensorflow:Assets written to: /tmp/saved_model/assets\n",
            "Time for the training is 114.01718521118164 sec,\n",
            "tf.Tensor([ 0.9509079 -0.8146242  1.7525027 -1.6756531], shape=(4,), dtype=float32)\n",
            "Model: \"model\"\n",
            "__________________________________________________________________________________________________\n",
            "Layer (type)                    Output Shape         Param #     Connected to                     \n",
            "==================================================================================================\n",
            "input_1 (InputLayer)            [(None, 4)]          0                                            \n",
            "__________________________________________________________________________________________________\n",
            "tf_op_layer_Shape (TensorFlowOp [(2,)]               0           input_1[0][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "tf_op_layer_strided_slice (Tens [()]                 0           tf_op_layer_Shape[0][0]          \n",
            "__________________________________________________________________________________________________\n",
            "tf_op_layer_random_normal/shape [(2,)]               0           tf_op_layer_strided_slice[0][0]  \n",
            "__________________________________________________________________________________________________\n",
            "tf_op_layer_RandomStandardNorma [(None, 4)]          0           tf_op_layer_random_normal/shape[0\n",
            "__________________________________________________________________________________________________\n",
            "tf_op_layer_mul (TensorFlowOpLa [(None, 4)]          0           tf_op_layer_RandomStandardNormal[\n",
            "__________________________________________________________________________________________________\n",
            "tf_op_layer_random_normal (Tens [(None, 4)]          0           tf_op_layer_mul[0][0]            \n",
            "__________________________________________________________________________________________________\n",
            "dense (Dense)                   (None, 4)            20          input_1[0][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "dense_1 (Dense)                 (None, 4)            20          tf_op_layer_random_normal[0][0]  \n",
            "__________________________________________________________________________________________________\n",
            "concatenate (Concatenate)       (None, 8)            0           dense[0][0]                      \n",
            "                                                                 dense_1[0][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "dense_2 (Dense)                 (None, 32)           288         concatenate[0][0]                \n",
            "__________________________________________________________________________________________________\n",
            "dense_3 (Dense)                 (None, 32)           1056        dense_2[0][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "dense_4 (Dense)                 (None, 4)            132         dense_3[0][0]                    \n",
            "==================================================================================================\n",
            "Total params: 1,516\n",
            "Trainable params: 1,516\n",
            "Non-trainable params: 0\n",
            "__________________________________________________________________________________________________\n",
            "Model: \"sequential\"\n",
            "_________________________________________________________________\n",
            "Layer (type)                 Output Shape              Param #   \n",
            "=================================================================\n",
            "dense_5 (Dense)              (None, 32)                288       \n",
            "_________________________________________________________________\n",
            "dense_6 (Dense)              (None, 1)                 32        \n",
            "=================================================================\n",
            "Total params: 320\n",
            "Trainable params: 320\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n",
            "CPU times: user 2min 10s, sys: 4.43 s, total: 2min 14s\n",
            "Wall time: 1min 54s\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Zw6Rt5z3Rjud",
        "colab_type": "code",
        "outputId": "0d68df06-e8d7-46cf-e697-fd0f1b971bbf",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 178
        }
      },
      "source": [
        "x = tf.random.normal((batch_size,n),dtype=tf.dtypes.float32)    #randomly sample input data (\"fake\" AE messages)\n",
        "x = x/tf.sqrt(2*tf.reduce_mean(tf.square(x)))\n",
        "#print(x)\n",
        "real_c = real_channel(x)\n",
        "fake_c = generator([x])\n",
        "tf.print(real_c.shape)\n",
        "tf.print(fake_c)\n",
        "generator.trainable = False\n",
        "tf.print(generator.trainable)\n",
        "#tf.debugging.check_numerics(fake_c,'message',name=None)"
      ],
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "TensorShape([1000, 4])\n",
            "[[0.0494147837 -0.13849245 -0.861038685 -1.29012895]\n",
            " [0.457019627 -0.31623438 -0.969341397 -1.7467283]\n",
            " [-0.323576927 0.252832741 0.326003969 0.1968804]\n",
            " ...\n",
            " [-0.0695377439 0.0657091811 -0.846659303 -0.881754398]\n",
            " [-1.86710691 0.621623039 1.03648734 0.112503931]\n",
            " [0.431880116 -0.408593565 -0.943853259 -0.27420336]]\n",
            "False\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "mmP50TkiAg-C",
        "colab_type": "text"
      },
      "source": [
        "## AE\n",
        "Die Idee sollte sein das Training auf den encoder und decoder einzuschränken. Jedoch soll **end-to-end** trainiert werden, hierfür sollte vllt eine art Funktion eingesetzt werden, welche über die GAN's Layer zurück geht.\n",
        "Muss ich hierfür die Layer nochmals einzeln definieren?\n",
        "\n",
        "\n",
        "***Vermutung: Der Ausgang hat die 8fache dimension des Eingangs-> daher nur 1/8 richtig oder 7/8 richtig*** \\\\\n",
        "**zu klären: was passiert in meinem AE dass sie dei dimension ver8-facht von (1000,8) zu (8000,n)**\n",
        "**Kontrollieren was der output von meinem GAN ist**\n",
        "**Add complexity for higher rubustness**\n",
        "\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FiuN3SZYpeTU",
        "colab_type": "code",
        "outputId": "030fb7a5-a67d-4031-e4cc-0a5a1fea45ef",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 624
        }
      },
      "source": [
        "def B_Ber(input_msg, msg):\n",
        "    '''Calculate the Batch Bit Error Rate'''\n",
        "    pred_error = tf.not_equal(tf.argmax(msg, 1), tf.argmax(input_msg, 1))\n",
        "    bber = tf.reduce_mean(tf.cast(pred_error, tf.float32))\n",
        "    return bber\n",
        "\n",
        "\n",
        "#def get_encoder():\n",
        "#  model = tf.keras.Sequential()\n",
        "#  model.add(tf.keras.layers.InputLayer(input_shape=[M]))\n",
        "#  model.add(tf.keras.layers.Dense(M,use_bias=True, activation='elu'))\n",
        "  #model.add(tf.keras.layers.Dense(M,use_bias=True, activation='relu'))\n",
        "#  model.add(tf.keras.layers.Dense(n,use_bias=False, activation=None))\n",
        "#  model.add(tf.keras.layers.Lambda(lambda x : tf.divide(x, tf.sqrt(2*tf.reduce_mean(tf.square(x))))))\n",
        "#  return model\n",
        "\n",
        "#def get_decoder():\n",
        "#  model = tf.keras.Sequential()\n",
        "#  model.add(tf.keras.layers.InputLayer(input_shape=[n]))\n",
        "#  model.add(tf.keras.layers.Dense(M,use_bias=True, activation='elu'))\n",
        "  #model.add(tf.keras.layers.Dense(M,use_bias=True, activation='relu'))\n",
        "#  model.add(tf.keras.layers.Dense(M,use_bias=False, activation='softmax'))\n",
        "#  return model\n",
        "\n",
        "#encoder = get_encoder()\n",
        "#decoder = get_decoder()\n",
        "\n",
        "\n",
        "   \n",
        "\n",
        "#def test_Model(x):\n",
        "#  y = encoder(x)\n",
        "#  y = generator([y,make_zero(y)])\n",
        "#  y = decoder(y)\n",
        "#  return y\n",
        "  \n",
        "#****************************************************  From  Rick Fritschek \"Communication-via-Autoencoder-TF2\"\n",
        "\n",
        "  \n",
        "\n",
        "\n",
        "gen_shape_layer = tf.keras.layers.Lambda(lambda x:tf.reshape(x,(tf.shape(x)[0],-1)))\n",
        "#gen_shape_layer = tf.keras.layers.Lambda(lambda x:tf.reshape(x,(tf.shape(x)[0],-1)))\n",
        "gen_shape_layer2 = tf.keras.layers.Lambda(lambda x:tf.reshape(x, shape=[-1,int(n/2),2]))\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "def decoder(input):\n",
        "  '''The Receiver'''\n",
        "  y = tf.reshape(input, shape=[-1,n])\n",
        "  y = tf.keras.layers.Dense(M, activation='relu')(y)\n",
        "  y = tf.keras.layers.Dense(M, activation=None)(y)\n",
        "  return y\n",
        "\n",
        "\n",
        "\n",
        "def encoder(input):\n",
        "  '''The transmitter'''\n",
        "  #low = np.sqrt(6.0/(2*M)) \n",
        "  #high = -np.sqrt(6.0/(2*M))\n",
        "  #W =tf.random.uniform((M,M), minval=low, maxval=high, dtype=tf.float32)    \n",
        "  #x = tf.nn.elu(tf.nn.embedding_lookup(W, input))\n",
        "  x = tf.keras.layers.Dense(n, activation=None)(input)\n",
        "  x = tf.reshape(x, shape=[-1,int(n/2),2])\n",
        "  #Average power normalization\n",
        "  x = x/tf.sqrt(2*tf.reduce_mean(tf.square(x))) \n",
        "  return x\n",
        "\n",
        "\n",
        "\n",
        "def generate_data_vector(length):\n",
        "  random_vector = tf.random.uniform(shape =(length,),minval=0,maxval=M, dtype=tf.dtypes.int32 ,seed=None,name=None)\n",
        "  random_hot_one_vector = tf.one_hot(random_vector, depth=M,on_value=1, off_value=0,axis=-1)\n",
        "  tf.print(random_hot_one_vector.shape)\n",
        "  return random_hot_one_vector\n",
        "\n",
        "bl = generate_data_vector(1000)\n",
        "bla =encoder(bl)            #ecoder: (4, 2, 2)\n",
        "blaa =tf.reshape(bla,(tf.shape(bla)[0],-1))\n",
        "blaaa = generator(blaa)\n",
        "blaaaa = tf.reshape(blaaa, shape=[-1,int(n/2),2])\n",
        "blaaaaa = decoder(blaaa)\n",
        "#blaaaaaa =  tf.reshape(blaaaaa, shape=[-1,2*n])\n",
        "\n",
        "print('encoder input',bl.shape)\n",
        "print('encoder output',bla.shape)\n",
        "print('reshaped generator input',blaa.shape)\n",
        "print('generator output',blaaa.shape)\n",
        "print('reshaped generator output',blaaaa.shape)\n",
        "print('decoder output',blaaaaa.shape)\n",
        "#print('reshape decoder',blaaaaaa.shape)\n",
        "\n",
        "#input1 = tf.keras.layers.Input(shape=(n,))\n",
        "#x1 = tf.keras.layers.Dense(n)(input1)\n",
        "\n",
        "'''\n",
        "EncIn = tf.keras.layers.Input(shape=(M,))#, dtype= tf.int32)\n",
        "e1 = tf.keras.layers.Dense(M, activation='relu')(EncIn)\n",
        "e2 = tf.keras.layers.Lambda(lambda x:tf.reshape(x, shape=[-1,int(n/2),2]))(e1)\n",
        "EncOut = tf.keras.layers.Lambda(lambda x: x/tf.sqrt(2*tf.reduce_mean(tf.square(x))))(e2)\n",
        "GenIn = tf.keras.layers.Lambda(lambda x:tf.reshape(x,(tf.shape(x)[0],-1)))(EncOut)\n",
        "GenOut = tf.keras.layers.Lambda(generator)(GenIn)\n",
        "DecIn = tf.keras.layers.Lambda(lambda x:tf.reshape(x, shape=[-1,int(n/2),2]))(GenOut)\n",
        "#out = tf.keras.layers.Lambda(decoder)(DecIn)\n",
        "d1 = tf.keras.layers.Lambda(lambda x:tf.reshape(x, shape=[-1,n]))(DecIn)\n",
        "d2 = tf.keras.layers.Dense(M, activation='relu')(d1)\n",
        "DecOut = tf.keras.layers.Dense(M, activation=None)(d2)\n",
        "'''\n",
        "EncIn = tf.keras.layers.Input(shape=(M,))#, dtype= tf.int32)\n",
        "e1 = tf.keras.layers.Dense(n, activation=None)\n",
        "e2 = tf.keras.layers.Lambda(lambda x:tf.reshape(x, shape=[-1,int(n/2),2]))\n",
        "EncOut = tf.keras.layers.Lambda(lambda x: x/tf.sqrt(2*tf.reduce_mean(tf.square(x))))\n",
        "GenIn = tf.keras.layers.Lambda(lambda x:tf.reshape(x,(tf.shape(x)[0],-1)))\n",
        "GenOut = tf.keras.layers.Lambda(generator)\n",
        "DecIn = tf.keras.layers.Lambda(lambda x:tf.reshape(x, shape=[-1,int(n/2),2]))\n",
        "#out = tf.keras.layers.Lambda(decoder)(DecIn)\n",
        "d1 = tf.keras.layers.Lambda(lambda x:tf.reshape(x, shape=[-1,n]))\n",
        "d2 = tf.keras.layers.Dense(n, activation='relu')\n",
        "DecOut = tf.keras.layers.Dense(M, activation='softmax')\n",
        "\n",
        "\n",
        "encoder =tf.keras.models.Sequential([EncIn,e1,e2,EncOut,GenIn]) \n",
        "decoder =tf.keras.models.Sequential([DecIn,d1,d2,DecOut])\n",
        "#AE = tf.keras.models.Model(inputs=EncIn, outputs=DecOut)\n",
        "AE = tf.keras.models.Sequential([encoder,generator,decoder])\n",
        "#AE.summary()\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "#encoder = keras.models.Sequential([\n",
        "#tf.keras.layers.InputLayer(input_shape=[M]),\n",
        "#tf.keras.layers.Dense(M, activation=\"relu\"),\n",
        "#tf.keras.layers.Dense(2*n, activation=None),\n",
        "#shape_layer,\n",
        "#norm_layer])\n",
        "\n",
        "\n",
        "\n",
        "#channel = keras.models.Sequential([channel_layer])\n",
        "\n",
        "#decoder = keras.models.Sequential([tf.keras.layers.InputLayer(input_shape=[2,n]),\n",
        "#shape_layer2,\n",
        "#tf.keras.layers.Dense(M, activation=\"relu\"),\n",
        "#tf.keras.layers.Dense(M, activation=\"softmax\")\n",
        "#])  \n",
        "  \n",
        "#encoder.summary()\n",
        "#generator.summary()\n",
        "#decoder.summary() \n",
        "\n",
        "\n",
        "\n",
        "#def get_AE():\n",
        "#  AE_model = tf.keras.Sequential()\n",
        "#  AE_model.add(tf.keras.layers.Lambda(encoder))\n",
        "#  AE_model.add(gen_shape_layer)\n",
        "#  AE_model.add(tf.keras.layers.Lambda(generator))\n",
        "#  AE_model.add(gen_shape_layer2)\n",
        "#  AE_model.add(tf.keras.layers.Lambda(decoder))\n",
        " # return AE_model\n",
        "  \n",
        "  \n",
        "  \n",
        "  \n",
        "  \n",
        "  \n",
        "  \n",
        "\n",
        "\n",
        "def generate_data_vector(length):\n",
        "  random_vector = tf.random.uniform(shape =(length,),minval=0,maxval=M, dtype=tf.dtypes.int32 ,seed=None,name=None)\n",
        "  random_hot_one_vector = tf.one_hot(random_vector, depth=M,on_value=1, off_value=0,axis=-1)\n",
        "  tf.print(random_hot_one_vector.shape)\n",
        "  return random_hot_one_vector\n",
        "\n",
        "data, test_data = generate_data_vector(10000000), generate_data_vector(10000)\n",
        "#print(data)\n",
        "\n",
        "#model = Autoencoder()\n",
        "#AE = get_AE()\n",
        "#tf.keras.optimizers.Adam(learning_rate=0.001, epsilon= 0.00001)\n",
        "AE.compile(optimizer='adam',loss=tf.keras.losses.categorical_crossentropy,metrics=[B_Ber])\n",
        "history = AE.fit(data, data, batch_size=200,steps_per_epoch=500, epochs=8)\n",
        "\n",
        "AE.summary()\n",
        "\n",
        "  "
      ],
      "execution_count": 46,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "TensorShape([1000, 16])\n",
            "encoder input (1000, 16)\n",
            "encoder output (1000, 2, 2)\n",
            "reshaped generator input (1000, 4)\n",
            "generator output (1000, 4)\n",
            "reshaped generator output (1000, 2, 2)\n",
            "decoder output (1000, 16)\n",
            "TensorShape([10000000, 16])\n",
            "TensorShape([10000, 16])\n",
            "Train on 10000000 samples\n",
            "Epoch 1/10\n",
            "   98000/10000000 [..............................] - ETA: 4:18 - loss: 2.0344 - B_Ber: 0.7033Epoch 2/10\n",
            "   98000/10000000 [..............................] - ETA: 2:15 - loss: 1.0686 - B_Ber: 0.2176Epoch 3/10\n",
            "   98400/10000000 [..............................] - ETA: 2:14 - loss: 0.5824 - B_Ber: 0.0680Epoch 4/10\n",
            "   99800/10000000 [..............................] - ETA: 2:18 - loss: 0.2677 - B_Ber: 0.0139Epoch 5/10\n",
            "   97600/10000000 [..............................] - ETA: 2:26 - loss: 0.1101 - B_Ber: 0.0021Epoch 6/10\n",
            "   98000/10000000 [..............................] - ETA: 2:31 - loss: 0.0558 - B_Ber: 0.0012Epoch 7/10\n",
            "  100000/10000000 [..............................] - ETA: 2:22 - loss: 0.0330 - B_Ber: 8.6000e-04Epoch 8/10\n",
            "   98600/10000000 [..............................] - ETA: 2:25 - loss: 0.0220 - B_Ber: 9.0264e-04Epoch 9/10\n",
            "   99200/10000000 [..............................] - ETA: 2:29 - loss: 0.0152 - B_Ber: 6.4516e-04Epoch 10/10\n",
            "   99800/10000000 [..............................] - ETA: 2:23 - loss: 0.0113 - B_Ber: 6.4128e-04Model: \"sequential_42\"\n",
            "_________________________________________________________________\n",
            "Layer (type)                 Output Shape              Param #   \n",
            "=================================================================\n",
            "sequential_40 (Sequential)   (None, None)              68        \n",
            "_________________________________________________________________\n",
            "model (Model)                (None, 4)                 1516      \n",
            "_________________________________________________________________\n",
            "sequential_41 (Sequential)   multiple                  100       \n",
            "=================================================================\n",
            "Total params: 1,684\n",
            "Trainable params: 168\n",
            "Non-trainable params: 1,516\n",
            "_________________________________________________________________\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "r-ZsnSNgM7g2",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "\n",
        "\n",
        "def analytic_channel(input): \n",
        "  #print(input.shape)\n",
        "  return input + tf.random.normal(tf.shape(input), mean=0.0, stddev=noise_std)\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "def real_transmision(test_data):\n",
        "  y = encoder(test_data)\n",
        "  y = generator(y)\n",
        "  y = decoder(y)\n",
        "  return y\n",
        "  #model = tf.keras.Sequential()\n",
        "  #model.add(encoder)\n",
        "  #model.add(tf.keras.layers.Lambda(generator))\n",
        "  #model.add(tf.keras.layers.Lambda(real_channel))\n",
        "  #model.add(decoder)\n",
        "  #return model\n",
        "\n",
        "def test_diff_eval(test_data, results):\n",
        "  diff = []\n",
        "  for i in range(tf.shape(test_data)[0]):\n",
        "    diff.append(tf.math.subtract(test_data[i,:], results[i,:]))\n",
        "  return diff\n",
        "    \n",
        "  \n",
        "real_AE = real_transmision(test_data)\n",
        "testTest = tf.dtypes.cast(real_AE + tf.constant(0.1,dtype=tf.float32,shape=tf.shape(real_AE)), tf.int32)\n",
        "\n",
        "diff_test =  test_diff_eval(test_data, testTest) \n",
        "#t = tf.math.subtract(test_data[1,:], real_AE[1,:])\n",
        "\n",
        "\n",
        "\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SntX-i_2J76v",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "5008cfb2-c739-47f4-b128-5453d21ed8dd"
      },
      "source": [
        "tf.print(sum(diff_test))"
      ],
      "execution_count": 52,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[3 6 10 ... 1 5 4]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "D5B2TUanPC5d",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 269
        },
        "outputId": "f1a88912-9d5e-4b9f-8f8f-60f0a178b4b8"
      },
      "source": [
        "tes_data = np.eye(M, dtype = int)\n",
        "coding= encoder.predict(tes_data)\n",
        "fig = plt.figure(figsize=(4,4))\n",
        "plt.plot(coding[:,0], coding[:,1],\"b.\")\n",
        "plt.gca().set_ylim(-2,2)\n",
        "plt.gca().set_xlim(-2,2)\n",
        "plt.show()"
      ],
      "execution_count": 53,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAARAAAAD8CAYAAAC/+/tYAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAADkhJREFUeJzt3W2sZWdZxvH/5dDyoTQWbZXSzlCM\nDTgiikxaJvXDaEHbhtCAkLQmQHlJE0MjJCRaJdFEPrRqglHbgA00gCGAAYpjKJYKTCpxwJ4209eh\nMjY2nbGxb9pCIDYDtx/2ns7hdJ85Z561Zu+99v7/ksnZL8/sZ510euV51rr3ulNVSFKLn5j1AUga\nLgNEUjMDRFIzA0RSMwNEUjMDRFKzzgGSZGuSrye5P8l9Sd47YUyS/HWSA0nuTvKrXeeVNHvP6+Ez\nDgPvr6o7k5wK3JHk1qq6f9WYi4Fzx3/OBz48/ilpwDqvQKrqkaq6c/z4u8B+4Kw1wy4FPlkj3wRO\nS3Jm17klzVYfK5BnJTkHeBXwrTVvnQU8vOr5wfFrj0z4jCuBKwFOOeWUV7/85S/v8xAlbcIdd9zx\neFWdsdG43gIkyQuAzwPvq6qnWz+nqm4AbgDYsWNHrays9HSEkjYryUObGdfLVZgkJzEKj09V1Rcm\nDDkEbF31/Ozxa5IGrI+rMAE+Buyvqg+tM2w38Lbx1ZjXAE9V1XO2L5KGpY8tzAXAW4F7kuwbv/ZH\nwDaAqvoIcDNwCXAA+D7wjh7mlTRjnQOkqr4BZIMxBbyn61yS5ouVqJKaGSCSmhkgkpoZIJKaGSCS\nmhkgkpoZIJKaGSCSmhkgkpoZIJKaGSCSmhkgkpoZIJKaGSCSmhkgkpoZIJKaGSCSmhkgkpr1dVf2\nG5M8muTedd7fleSpJPvGf/64j3klzVZffWE+DlwHfPIYY/6lql7f03yS5kAvK5Cqug14so/PkjQc\n0zwHsjPJXUm+nOQXpzivpBOk1964x3An8JKq+l6SS4AvAudOGri6N+62bdumdHiSWkxlBVJVT1fV\n98aPbwZOSnL6OmNvqKodVbXjjDM27O0raYamEiBJXjRugUmS88bzPjGNuSWdOL1sYZJ8GtgFnJ7k\nIPAnwEnwbGvLNwO/m+Qw8APgsnG3OkkD1kuAVNXlG7x/HaPLvJIWiJWokpoZIJKaGSCSmhkgkpoZ\nIJKaGSCSmhkgkpoZIJKaGSCSmhkgkpoZIJKaGSCSmhkgkpoZIJKaGSCaur174ZprRj81bNO6J6oE\njELjwgvhmWfg5JPhq1+FnTtnfVRq5QpEU7Vnzyg8fvjD0c89e2Z9ROrCANFU7do1Wnls2TL6uWvX\nrI9IXbiF0VTt3DnatuzZMwoPty/D1tdNlW8EXg88WlWvmPB+gL8CLgG+D1xRVXf2MbeGZ+dOg2NR\n9LWF+Thw0THev5hRI6lzGTWN+nBP80qaoWn1xr0U+GSNfBM4LcmZfcwtaXamdRL1LODhVc8Pjl97\njiRXJllJsvLYY49N5eAktZm7qzC2tpSGY1oBcgjYuur52ePXJA3YtAJkN/C2jLwGeKqqHpnS3JJO\nkGn1xr2Z0SXcA4wu476jj3klzda0euMW8J4+5pI0P+buJKqk4TBAJDUzQCQ1M0A0KN6MaL74bVwN\nhjcjmj+uQDQY3oxo/hggGgxvRjR/3MJoMLwZ0fwxQDQo3oxovriFkdTMAJHUzACRGliPMuI5EOk4\nWY9ylCsQ6ThZj3KUASIdJ+tRjnILIx0n61GOMkCkBtajjLiFkdSslwBJclGSB5IcSHL1hPevSPJY\nkn3jP+/uY15Js9V5C5NkC3A98DpGDaNuT7K7qu5fM/SzVXVV1/mkIdq7dzHPmfRxDuQ84EBVPQiQ\n5DOMWlmuDRBpKS1y3UgfW5jNtq387SR3J/lckq0T3gdsbanFs8h1I9M6ifqPwDlV9UrgVuAT6w20\ntaUWzSLXjfSxhdmwbWVVPbHq6UeBP+9hXmkQFrlupI8AuR04N8lLGQXHZcDvrB6Q5MxVrSzfAOzv\nYV5pMBa1bqRzgFTV4SRXAbcAW4Abq+q+JH8KrFTVbuD3krwBOAw8CVzRdV5Js5dR18n5tGPHjlpZ\nWZn1YcydRb0kqPmR5I6q2rHROEvZB2aRLwlqeCxlH5hFviSo4TFABmaRLwlqeNzCDMwiXxLU8Bgg\nA7SolwQ1PG5hJDUzQCQ1M0AkNTNAJDUzQCQ1M0AkNTNAJDUzQCQ1M0AkNTNAJDUzQCQ1M0AkNTNA\nJDWbVmvL5yf57Pj9byU5p495Jc1W5wBZ1dryYmA7cHmS7WuGvQv4n6r6eeAvgT/rOq+k2etjBfJs\na8uqegY40tpytUs52kzqc8CFSdLD3NJx27sXrrlm9FPd9HFDoUmtLc9fb8y4DcRTwE8Dj6/9sCRX\nAlcCbNu2rYfDk47yptT9mruTqLa2VBcbrS68KXW/ptLactWYg0meB/wk8ARSjzazujhyU+ojY7wp\ndTd9rECebW2Z5GRGrS13rxmzG3j7+PGbga/VPHe00iBtZnVx5KbUH/yg25c+TKu15ceAv0tygFFr\ny8u6ziuttdnVhTel7o+tLbVQbPvZD1tbaim5upiuubsKI2k4DBBJzQwQTZ2VoIvDcyCaKitBF4sr\nEE2VlaCLxQDRCbHeNuVIrcaWLVaCLgK3MOrdsbYpRypBrdVYDAaIejdpm7I6KKzVWBxuYdQ7tynL\nwxWIeuc2ZXkYIDoh3KYsB7cwkpoZIJKaGSDHyTJs6SjPgRwHy7ClH+cK5DhYhq1JlnlV6grkOHhD\nXq217KvSTiuQJD+V5NYk3xn/fOE6436YZN/4z9obLg+GN+TVWsu+Ku26Arka+GpVXTvuiXs18AcT\nxv2gqn6l41xzwfoGrbbsq9KuAXIpsGv8+BPAHiYHiLSQlr3qttNd2ZP8b1WdNn4cRg20T5sw7jCw\nDzgMXFtVXzzGZ65ubfnqhx56qPn4JLXp7a7sSf4ZeNGEtz6w+klVVZL10uglVXUoyc8BX0tyT1X9\nx6SBVXUDcAOM2jpsdHySZmfDAKmq1673XpL/TnJmVT2S5Ezg0XU+49D454NJ9gCvAiYGiKTh6FoH\nsrpl5duBf1g7IMkLkzx//Ph04ALg/o7zSpoDXQPkWuB1Sb4DvHb8nCQ7knx0POYXgJUkdwFfZ3QO\nxADRwrCQrFFVPQFcOOH1FeDd48f/CvxSl3mkeWUhmaRmy15IZoBIHSz77Rv9LozUwbIXkhkgUkfL\n/PUGtzCSmhkgkpoZIJKaGSBLbpmLoNSdJ1GX2LIXQak7VyBLbNmLoNSdAbLElr0ISt25hVliy14E\npe4MkCW3zEVQ6s4tjKRmBoikZgaIpGYGiAbLIrjZ8ySqBskiuPnQtbXlW5Lcl+RHSdbtIZHkoiQP\nJDkw7mAndWIR3HzouoW5F3gTcNt6A5JsAa4HLga2A5cn2d5xXi05i+DmQ9ebKu8HGDWlW9d5wIGq\nenA89jOMWmJ6Z3Y1swhuPkzjHMhZwMOrnh8Ezl9v8JrWlif2yDRoFsHNXqfWllX1nEZSXdnaUhqO\nTq0tN+kQsHXV87PHr0kauGnUgdwOnJvkpUlOBi5j1BJTc8a6Ch2vTudAkrwR+BvgDOBLSfZV1W8l\neTHw0aq6pKoOJ7kKuAXYAtxYVfd1PnL1yroKteh6FeYm4KYJr/8XcMmq5zcDN3eZSyfWpLoKA0Qb\nsZRdgHUVamMpuwDrKtTGANGzrKvQ8XILI6mZASKpmQEiqZkBIqmZASKpmQEiqZkBIqmZASKpmQEi\nqZkBooXiLQmmy1J2LQxvSTB9rkC0MGz1MH0GiBaGtySYPrcwWhjekmD6DBAtFG9JMF3Tam35n0nu\nSbIvyUqXOSXNj64rkCOtLf92E2N/vaoe7zifpDkyjdaWkhbUtK7CFPCVJHeMW1dKWgDTam35a1V1\nKMnPALcm+XZV3bbOfPbGlQZiGq0tqapD45+PJrkJOA+YGCD2xpWG44RvYZKckuTUI4+B32R08lXS\nwHW9jPvGJAeBnYxaW94yfv3FSY50ovtZ4BtJ7gL+DfhSVf1Tl3klzYcT3tqyqh4EfrnLPJLmk9+F\nkdTMAJHUzACR1MwAkdTMAJHUzACR1MwAkdTMAJHUzACR1MwAkdTMAJHUzACR1MwAkdTMAJHUzACR\n1MwAkdTMAJHUzACR1MwAkdSs602V/yLJt5PcneSmJKetM+6iJA8kOZDk6i5zSpofXVcgtwKvqKpX\nAv8O/OHaAUm2ANcDFwPbgcuTbO84r6Q50ClAquorVXV4/PSbwNkThp0HHKiqB6vqGeAzwKVd5pU0\nHzq1dVjjncBnJ7x+FvDwqucHgfPX+5DVrS2B/0uyqE2oTgcen/VBnED+fsP2ss0M6qU3bpIPAIeB\nTx3PEU6yurVlkpWq2tH1M+fRIv9u4O83dElWNjOuc2/cJFcArwcurKpJvWwPAVtXPT97/Jqkget6\nFeYi4PeBN1TV99cZdjtwbpKXJjkZuAzY3WVeSfOh61WY64BTgVuT7EvyEfjx3rjjk6xXAbcA+4G/\nr6r7Nvn5N3Q8vnm2yL8b+PsN3aZ+v0zedUjSxqxEldTMAJHUbK4DZLOl8kOV5C1J7kvyoyQLc0lw\nkb+6kOTGJI8uYn1Skq1Jvp7k/vG/y/du9HfmOkDYRKn8wN0LvAm4bdYH0pcl+OrCx4GLZn0QJ8hh\n4P1VtR14DfCejf7bzXWAbLJUfrCqan9VPTDr4+jZQn91oapuA56c9XGcCFX1SFXdOX78XUZXTc86\n1t+Z6wBZ453Al2d9ENrQpK8uHPMfoeZPknOAVwHfOta4Pr8L02TapfLTtpnfT5onSV4AfB54X1U9\nfayxMw+QHkrl59pGv98C8qsLA5bkJEbh8amq+sJG4+d6C7PJUnnNF7+6MFBJAnwM2F9VH9rM35nr\nAGGdUvlFkeSNSQ4CO4EvJbll1sfUVcevLsy9JJ8G9gIvS3IwybtmfUw9ugB4K/Ab4//f9iW55Fh/\nwVJ2Sc3mfQUiaY4ZIJKaGSCSmhkgkpoZIJKaGSCSmhkgkpr9P0G/SPOJNIiwAAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 288x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "R8o3nqP_0OTK",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 269
        },
        "outputId": "28da122c-6264-4cdf-e1eb-5a54412b0d59"
      },
      "source": [
        "tes_data = np.eye(M, dtype = int)\n",
        "coding= AE.predict(tes_data)\n",
        "fig = plt.figure(figsize=(4,4))\n",
        "plt.plot(coding[:,0], coding[:,1],\"b.\")\n",
        "plt.gca().set_ylim(-2,2)\n",
        "plt.gca().set_xlim(-2,2)\n",
        "plt.show()"
      ],
      "execution_count": 50,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAARAAAAD8CAYAAAC/+/tYAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAADTZJREFUeJzt3X2snvVdx/H3x66dCSMyBRmWdmBs\nhlWnbCewE/zjaDcDZAE3R1JMpuwhTQzELVli0CXT+A8zJjNZIJvNIIAhg2UbWkMnqx1NXSzIKSkP\npcNVoqG1ypOBLcy1ZV//uO/isZwnf9d97ofD+5XcuR7u37l+vyuUT66n+/qmqpCkFj826gFImlwG\niKRmBoikZgaIpGYGiKRmBoikZp0DJMmGJA8keTLJwSSfmKdNknw+yeEkjyV5V9d+JY3emwawjZPA\np6rqkSRnAvuT7KqqJ+e0uQLY1P9cCnyhP5U0wTofgVTVsap6pD//PeAQsP60ZlcDd1bPg8BZSc7r\n2rek0RrEEchrklwAXAw8dNpX64Fn5iwf6a87Ns82tgHbAM4444x3X3TRRYMcoqRl2L9///NVdc5S\n7QYWIEneAnwN+GRVvdy6naraDmwHmJqaqtnZ2QGNUNJyJfm35bQbyF2YJGvphcddVfX1eZocBTbM\nWT6/v07SBBvEXZgAtwKHqupzCzTbAfxO/27Me4CXqup1py+SJssgTmEuAz4MPJ7kQH/dHwEbAarq\ni8BO4ErgMPAK8JEB9CtpxDoHSFV9G8gSbQq4vmtfksaLT6JKamaASGpmgEhqZoBIamaASGpmgEhq\nZoBIamaASGpmgEhqZoBIamaASGpmgEhqZoBIamaASGpmgEhqZoBIamaASGpmgEhqNqi3st+W5Nkk\nTyzw/UySl5Ic6H8+M4h+JY3WoOrC3A7cDNy5SJt/qKr3D6g/SWNgIEcgVbUXeHEQ25I0OYZ5DWQ6\nyaNJvpHkF4bYr6QVMtDauIt4BHh7VX0/yZXAXwOb5ms4tzbuxo0bhzQ8SS2GcgRSVS9X1ff78zuB\ntUnOXqDt9qqaqqqpc85ZsravpBEaSoAkeVu/BCZJLun3+8Iw+pa0cgZyCpPky8AMcHaSI8AfA2vh\ntdKWHwJ+L8lJ4AfA1n61OkkTbCABUlXXLvH9zfRu80paRXwSVVIzA0RSMwNEUjMDRFIzA0RSMwNE\nUjMDRFIzA0RSMwNEUjMDRFIzA0RSMwNEUjMDRFIzA0RSMwNEUjMDRFIzA0RSMwNEUjMDRFKzYdXG\nTZLPJzmc5LEk7xpEv5JGa1BHILcDly/y/RX0Ckltolc06gsD6ldjaN8+uOmm3lSr26Deyr43yQWL\nNLkauLNfyuHBJGclOa+qjg2if42PfftgyxY4fhzWrYPdu2F6etSj0koZ1jWQ9cAzc5aP9Ne9TpJt\nSWaTzD733HNDGZwGZ8+eXni8+mpvumfPqEeklTR2F1EtbTnZZmZ6Rx5r1vSmMzOjHpFW0rCKax8F\nNsxZPr+/TqvM9HTvtGXPnl54ePqyug0rQHYANyS5G7gUeMnrH6vX9LTB8UYxrNq4O4ErgcPAK8BH\nBtGvpNEaVm3cAq4fRF+SxsfYXUSVNDkMEEnNDBBJzQwQSc0MEEnNDBBJzQwQSc0MEEnNDBBJzQwQ\nSc0MEEnNDBBJzQwQSc0MEEnNDBBJzQwQSc0MEEnNDBBJzQZV2vLyJE/1S1feOM/31yV5LsmB/ufj\ng+hX0mh1fidqkjXALcD76BWMejjJjqp68rSm91TVDV37kzQ+BnEEcglwuKqerqrjwN30SllKWuUG\nESDLLVv5W0keS/LVJBvm+R6wtKU0SYZ1EfVvgQuq6p3ALuCOhRpa2lKaHIMIkCXLVlbVC1X1w/7i\nl4B3D6BfSSM2iAB5GNiU5MIk64Ct9EpZvibJeXMWrwIODaBfSSPW+S5MVZ1McgNwP7AGuK2qDib5\nU2C2qnYAv5/kKuAk8CJwXdd+JY1eelUnx9PU1FTNzs6OehjSG06S/VU1tVQ7n0SV1MwAkdTMAJHU\nzACR1MwAkdTMAJHUzACR1MwAkdTMAJHUzACR1MwAkdTMAJHUzACR1MwAkdTMAJHUzACR1MwAkdTM\nAJHUbFilLd+c5J7+9w8luWAQ/Uoarc4BMqe05RXAZuDaJJtPa/Yx4L+q6ueAvwD+rGu/Gl/r1kHS\nm65G+/bBTTf1pm90nd/KzpzSlgBJTpW2nFsb92rgT/rzXwVuTpIa5zc6q8m6dXDiRG/+xIne8vHj\nox3TIO3bB1u29PZp3TrYvRump0c9qtEZVmnL19pU1UngJeCn5tuYpS0n26nwWGh50u3Z0wuPV1/t\nTffsGfWIRmvsLqJa2nKyrV27+PKkm5npHXmsWdObzsyMekSjNYhTmCVLW85pcyTJm4CfAF4YQN8a\nM6cO7U+c6IXHajp9gd7pyu7dvSOPmZk39ukLDCZAXittSS8otgK/fVqbHcDvAvuADwHf8vrH6rXa\nQuN009MGxynDKm15K/BXSQ7TK225tWu/kkZvEEcgVNVOYOdp6z4zZ/6/gWsG0Zek8TF2F1ElTQ4D\nRFIzA0RSMwNEUjMDRFIzA0RSMwNEUjMDRFIzA0RSMwNEUjMDRFIzA0RSMwNEUjMDRFIzA0RSMwNE\nUjMDRFIzA0RSs04BkuQnk+xK8t3+9K0LtHs1yYH+Z0eXPiWNj65HIDcCu6tqE7C7vzyfH1TVr/Q/\nV3XsU9KY6BogVwN39OfvAH6z4/YkTZCuAXJuVR3rz/8HcO4C7X68X67ywSSLhoylLaXJsWRZhyR/\nD7xtnq8+PXehqirJQsWi3l5VR5P8LPCtJI9X1b/M17CqtgPbAaampiw+JY2xJQOkqt670HdJ/jPJ\neVV1LMl5wLMLbONof/p0kj3AxcC8ASJpcnQ9hTlVspL+9G9Ob5DkrUne3J8/G7gMeLJjv5LGQNcA\n+SzwviTfBd7bXybJVJIv9dv8PDCb5FHgAeCzVWWASKtAp9KWVfUCsGWe9bPAx/vz/wj8Upd+JI0n\nn0SV1MwAkdTMAJHUzACR1MwAkdTMAJHUzACR1MwAkdTMAJHUzACR1MwAkdTMAJHUzACR1MwAkdTM\nAJHUzACR1MwAkdTMAJHUrGtpy2uSHEzyoyRTi7S7PMlTSQ4nWah6naQJ0/UI5Angg8DehRokWQPc\nAlwBbAauTbK5Y7+SxkDXlyofAkiyWLNLgMNV9XS/7d30SmL6ZnZpwg3jGsh64Jk5y0f66+ZlaUtp\ncnQqbVlVrysk1ZWlLaXJ0am05TIdBTbMWT6/v07ShBvGKczDwKYkFyZZB2ylVxJT0oTrehv3A0mO\nANPAfUnu76//mSQ7AarqJHADcD9wCPhKVR3sNmxJ46DrXZh7gXvnWf/vwJVzlncCO7v0JWn8+CSq\npGYGiKRmBoikZgaIpGYGiKRmBoikZgaIpGYGiKRmBoikZgaIpGYGiKRmBoikZgaIpGYGiKRmBoik\nZgaIpGYGiKRmBoikZsMqbfmvSR5PciDJbJc+JY2PTu9E5X9LW/7lMtr+WlU937E/SWNkGKUtJa1S\nw7oGUsA3k+xPsm1IfUpaYcMqbfmrVXU0yU8Du5J8p6r2LtDfNmAbwMaNG5e5eUmjMIzSllTV0f70\n2ST3ApcA8waItXGlybHipzBJzkhy5ql54DfoXXyVNOFWvLQlcC7w7SSPAv8E3FdVf9elX0njYcVL\nW1bV08Avd+lH0njySVRJzQwQSc0MEEnNDBBJzQwQSc0MEEnNDBBJzQwQSc0MEEnNDBBJzQwQSc0M\nEEnNDBBJzQwQSc0MEEnNDBBJzQwQSc0MEEnNDBBJzbq+VPnPk3wnyWNJ7k1y1gLtLk/yVJLDSW7s\n0qek8dH1CGQX8ItV9U7gn4E/PL1BkjXALcAVwGbg2iSbO/YraQx0CpCq+mZVnewvPgicP0+zS4DD\nVfV0VR0H7gau7tKvpPHQqazDaT4K3DPP+vXAM3OWjwCXLrSRuaUtgR8mWa1FqM4Gnh/1IFaQ+zfZ\n3rGcRgOpjZvk08BJ4K7/zwjnM7e0ZZLZqprqus1xtJr3Ddy/SZdkdjntOtfGTXId8H5gS1XNV8v2\nKLBhzvL5/XWSJlzXuzCXA38AXFVVryzQ7GFgU5ILk6wDtgI7uvQraTx0vQtzM3AmsCvJgSRfhP9b\nG7d/kfUG4H7gEPCVqjq4zO1v7zi+cbaa9w3cv0m3rP3L/GcdkrQ0n0SV1MwAkdRsrANkuY/KT6ok\n1yQ5mORHSVbNLcHV/NOFJLcleXY1Pp+UZEOSB5I82f93+Yml/masA4RlPCo/4Z4APgjsHfVABuUN\n8NOF24HLRz2IFXIS+FRVbQbeA1y/1H+7sQ6QZT4qP7Gq6lBVPTXqcQzYqv7pQlXtBV4c9ThWQlUd\nq6pH+vPfo3fXdP1ifzPWAXKajwLfGPUgtKT5frqw6D9CjZ8kFwAXAw8t1m6Qv4VpMuxH5YdtOfsn\njZMkbwG+Bnyyql5erO3IA2QAj8qPtaX2bxXypwsTLMlaeuFxV1V9fan2Y30Ks8xH5TVe/OnChEoS\n4FbgUFV9bjl/M9YBwgKPyq8WST6Q5AgwDdyX5P5Rj6mrjj9dGHtJvgzsA96R5EiSj416TAN0GfBh\n4Nf7/78dSHLlYn/go+ySmo37EYikMWaASGpmgEhqZoBIamaASGpmgEhqZoBIavY/+afnMt1Uw0wA\nAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 288x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "6SR4RrE3nqTc",
        "colab_type": "text"
      },
      "source": [
        "## Create and train model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Xi_IcVrbnS1_",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}